@moontaiworks/fanbox-dl 1.6.0 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +51 -37
  2. package/dist/cjs/downloader/asset/download.d.ts +3 -1
  3. package/dist/cjs/downloader/asset/download.d.ts.map +1 -1
  4. package/dist/cjs/downloader/asset/download.js +11 -14
  5. package/dist/cjs/downloader/asset/download.js.map +1 -1
  6. package/dist/cjs/downloader/cli/options.d.ts +2 -1
  7. package/dist/cjs/downloader/cli/options.d.ts.map +1 -1
  8. package/dist/cjs/downloader/cli/options.js +3 -1
  9. package/dist/cjs/downloader/cli/options.js.map +1 -1
  10. package/dist/cjs/downloader/creator/discover-posts.d.ts.map +1 -1
  11. package/dist/cjs/downloader/creator/discover-posts.js +5 -3
  12. package/dist/cjs/downloader/creator/discover-posts.js.map +1 -1
  13. package/dist/cjs/downloader/creator/sync.d.ts +2 -1
  14. package/dist/cjs/downloader/creator/sync.d.ts.map +1 -1
  15. package/dist/cjs/downloader/creator/sync.js +25 -26
  16. package/dist/cjs/downloader/creator/sync.js.map +1 -1
  17. package/dist/cjs/downloader/fs/filesystem.d.ts +3 -0
  18. package/dist/cjs/downloader/fs/filesystem.d.ts.map +1 -1
  19. package/dist/cjs/downloader/fs/filesystem.js +27 -2
  20. package/dist/cjs/downloader/fs/filesystem.js.map +1 -1
  21. package/dist/cjs/downloader/index.d.ts +1 -1
  22. package/dist/cjs/downloader/index.d.ts.map +1 -1
  23. package/dist/cjs/downloader/index.js +13 -16
  24. package/dist/cjs/downloader/index.js.map +1 -1
  25. package/dist/cjs/downloader/manifest/creator-manager.d.ts +3 -0
  26. package/dist/cjs/downloader/manifest/creator-manager.d.ts.map +1 -1
  27. package/dist/cjs/downloader/manifest/creator-manager.js +24 -1
  28. package/dist/cjs/downloader/manifest/creator-manager.js.map +1 -1
  29. package/dist/cjs/downloader/manifest/creator.d.ts +6 -0
  30. package/dist/cjs/downloader/manifest/creator.d.ts.map +1 -1
  31. package/dist/cjs/downloader/manifest/creator.js +11 -1
  32. package/dist/cjs/downloader/manifest/creator.js.map +1 -1
  33. package/dist/cjs/downloader/post/article.d.ts.map +1 -1
  34. package/dist/cjs/downloader/post/article.js +25 -22
  35. package/dist/cjs/downloader/post/article.js.map +1 -1
  36. package/dist/cjs/downloader/post/sync.d.ts +3 -2
  37. package/dist/cjs/downloader/post/sync.d.ts.map +1 -1
  38. package/dist/cjs/downloader/post/sync.js +123 -85
  39. package/dist/cjs/downloader/post/sync.js.map +1 -1
  40. package/dist/cjs/downloader/post/verify.d.ts +9 -0
  41. package/dist/cjs/downloader/post/verify.d.ts.map +1 -0
  42. package/dist/cjs/downloader/post/verify.js +64 -0
  43. package/dist/cjs/downloader/post/verify.js.map +1 -0
  44. package/dist/esm/downloader/asset/download.d.ts +3 -1
  45. package/dist/esm/downloader/asset/download.d.ts.map +1 -1
  46. package/dist/esm/downloader/asset/download.js +12 -15
  47. package/dist/esm/downloader/asset/download.js.map +1 -1
  48. package/dist/esm/downloader/cli/options.d.ts +2 -1
  49. package/dist/esm/downloader/cli/options.d.ts.map +1 -1
  50. package/dist/esm/downloader/cli/options.js +3 -1
  51. package/dist/esm/downloader/cli/options.js.map +1 -1
  52. package/dist/esm/downloader/creator/discover-posts.d.ts.map +1 -1
  53. package/dist/esm/downloader/creator/discover-posts.js +5 -3
  54. package/dist/esm/downloader/creator/discover-posts.js.map +1 -1
  55. package/dist/esm/downloader/creator/sync.d.ts +2 -1
  56. package/dist/esm/downloader/creator/sync.d.ts.map +1 -1
  57. package/dist/esm/downloader/creator/sync.js +25 -26
  58. package/dist/esm/downloader/creator/sync.js.map +1 -1
  59. package/dist/esm/downloader/fs/filesystem.d.ts +3 -0
  60. package/dist/esm/downloader/fs/filesystem.d.ts.map +1 -1
  61. package/dist/esm/downloader/fs/filesystem.js +24 -2
  62. package/dist/esm/downloader/fs/filesystem.js.map +1 -1
  63. package/dist/esm/downloader/index.d.ts +1 -1
  64. package/dist/esm/downloader/index.d.ts.map +1 -1
  65. package/dist/esm/downloader/index.js +13 -16
  66. package/dist/esm/downloader/index.js.map +1 -1
  67. package/dist/esm/downloader/manifest/creator-manager.d.ts +3 -0
  68. package/dist/esm/downloader/manifest/creator-manager.d.ts.map +1 -1
  69. package/dist/esm/downloader/manifest/creator-manager.js +24 -1
  70. package/dist/esm/downloader/manifest/creator-manager.js.map +1 -1
  71. package/dist/esm/downloader/manifest/creator.d.ts +6 -0
  72. package/dist/esm/downloader/manifest/creator.d.ts.map +1 -1
  73. package/dist/esm/downloader/manifest/creator.js +11 -1
  74. package/dist/esm/downloader/manifest/creator.js.map +1 -1
  75. package/dist/esm/downloader/post/article.d.ts.map +1 -1
  76. package/dist/esm/downloader/post/article.js +25 -22
  77. package/dist/esm/downloader/post/article.js.map +1 -1
  78. package/dist/esm/downloader/post/sync.d.ts +3 -2
  79. package/dist/esm/downloader/post/sync.d.ts.map +1 -1
  80. package/dist/esm/downloader/post/sync.js +123 -85
  81. package/dist/esm/downloader/post/sync.js.map +1 -1
  82. package/dist/esm/downloader/post/verify.d.ts +9 -0
  83. package/dist/esm/downloader/post/verify.d.ts.map +1 -0
  84. package/dist/esm/downloader/post/verify.js +61 -0
  85. package/dist/esm/downloader/post/verify.js.map +1 -0
  86. package/package.json +1 -1
@@ -0,0 +1,64 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.verifyCompletePost = verifyCompletePost;
4
+ const promises_1 = require("node:fs/promises");
5
+ const filesystem_js_1 = require("../fs/filesystem.js");
6
+ async function verifyCompletePost({ logger }, manifest, postSummary) {
7
+ const assets = Object.fromEntries(await Promise.all(Object.entries(manifest.assets).map(async ([id, asset]) => [
8
+ id,
9
+ asset ? await verifyAsset({ logger }, asset, postSummary) : asset,
10
+ ])));
11
+ const hasFailedAsset = Object.values(assets).some((asset) => asset && asset.status !== "complete");
12
+ return {
13
+ ...manifest,
14
+ assets,
15
+ status: hasFailedAsset ? "partial" : "complete",
16
+ };
17
+ }
18
+ function expectedModifiedTime(asset, postSummary) {
19
+ if (asset.expectedTime) {
20
+ const modifiedTime = (0, filesystem_js_1.normalizeFileTimestamp)(new Date(asset.expectedTime));
21
+ if (!Number.isNaN(modifiedTime.getTime()))
22
+ return modifiedTime;
23
+ }
24
+ return (0, filesystem_js_1.normalizeFileTimestamp)(new Date(new Date(postSummary.updatedDatetime).getTime() +
25
+ asset.contentIndex * 1000));
26
+ }
27
+ function failedAsset(asset, error) {
28
+ return {
29
+ ...asset,
30
+ error,
31
+ status: "failed",
32
+ };
33
+ }
34
+ function obsoleteAsset(asset, error) {
35
+ return {
36
+ ...asset,
37
+ error,
38
+ status: "obsolete",
39
+ };
40
+ }
41
+ async function verifyAsset({ logger }, asset, postSummary) {
42
+ if (asset.status !== "complete") {
43
+ return asset;
44
+ }
45
+ if (asset.contentIndex === undefined) {
46
+ return obsoleteAsset(asset, "content index missing");
47
+ }
48
+ if (!(await (0, filesystem_js_1.exists)(asset.path))) {
49
+ return failedAsset(asset, "file missing");
50
+ }
51
+ const actualSha256 = await (0, filesystem_js_1.hashFile)(asset.path);
52
+ if (actualSha256 !== asset.sha256) {
53
+ return failedAsset(asset, "sha256 mismatch");
54
+ }
55
+ const expectedTime = expectedModifiedTime(asset, postSummary);
56
+ const { mtimeMs } = await (0, promises_1.stat)(asset.path);
57
+ if ((0, filesystem_js_1.normalizeFileTimestamp)(new Date(mtimeMs)).getTime() !==
58
+ expectedTime.getTime()) {
59
+ logger.debug(`Repairing modified time of verified asset ${asset.path} from ${new Date(mtimeMs).toISOString()} to ${expectedTime.toISOString()}`);
60
+ await (0, promises_1.utimes)(asset.path, expectedTime, expectedTime);
61
+ }
62
+ return { ...asset, expectedTime: (0, filesystem_js_1.formatFileTimestamp)(expectedTime) };
63
+ }
64
+ //# sourceMappingURL=verify.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"verify.js","sourceRoot":"","sources":["../../../../src/downloader/post/verify.ts"],"names":[],"mappings":";;AAoBA,gDAyBC;AA7CD,+CAAgD;AAKhD,uDAK6B;AAUtB,KAAK,UAAU,kBAAkB,CACtC,EAAE,MAAM,EAAuB,EAC/B,QAA0B,EAC1B,WAAwB;IAExB,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAC/B,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,CACjC,KAAK,EAAE,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CACpB;QACE,EAAE;QACF,KAAK,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK;KAChB,CACtD,CACF,CACF,CAAC;IACF,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAC/C,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU,CAChD,CAAC;IAEF,OAAO;QACL,GAAG,QAAQ;QACX,MAAM;QACN,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU;KAChD,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,KAAwB,EACxB,WAAwB;IAExB,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;QACvB,MAAM,YAAY,GAAG,IAAA,sCAAsB,EAAC,IAAI,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC;QAC1E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;YAAE,OAAO,YAAY,CAAC;IACjE,CAAC;IAED,OAAO,IAAA,sCAAsB,EAC3B,IAAI,IAAI,CACN,IAAI,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE;QAC7C,KAAK,CAAC,YAAa,GAAG,IAAK,CAC9B,CACF,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAClB,KAAwB,EACxB,KAAa;IAEb,OAAO;QACL,GAAG,KAAK;QACR,KAAK;QACL,MAAM,EAAE,QAAQ;KACjB,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,KAAwB,EACxB,KAAa;IAEb,OAAO;QACL,GAAG,KAAK;QACR,KAAK;QACL,MAAM,EAAE,UAAU;KACnB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,EAAE,MAAM,EAAuB,EAC/B,KAAwB,EACxB,WAAwB;IAExB,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACrC,OAAO,aAAa,CAAC,KAAK,EAAE,uBAAuB,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,CAAC,CAAC,MAAM,IAAA,sBAAM,EAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAChC,OAAO,WAAW,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,IAAA,wBAAQ,EAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChD,IAAI,YAAY,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QAClC,OAAO,WAAW,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAC9D,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAA,eAAI,EAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,IACE,IAAA,sCAAsB,EAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE;QACnD,YAAY,CAAC,OAAO,EAAE,EACtB,CAAC;QACD,MAAM,CAAC,KAAK,CACV,6CAA6C,KAAK,CAAC,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,OAAO,YAAY,CAAC,WAAW,EAAE,EAAE,CACnI,CAAC;QACF,MAAM,IAAA,iBAAM,EAAC,KAAK,CAAC,IAAI,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC;IACvD,CAAC;IAED,OAAO,EAAE,GAAG,KAAK,EAAE,YAAY,EAAE,IAAA,mCAAmB,EAAC,YAAY,CAAC,EAAE,CAAC;AACvE,CAAC"}
@@ -10,9 +10,11 @@ interface DownloadAssetOptions {
10
10
  destination: string;
11
11
  fallbackDateTime: string;
12
12
  mediaContent: MediaContent;
13
+ timeOffset: number;
13
14
  }
14
- export declare function downloadAsset({ headers, logger, transport }: DownloadAssetDeps, { destination, fallbackDateTime, mediaContent }: DownloadAssetOptions): Promise<{
15
+ export declare function downloadAsset({ headers, logger, transport }: DownloadAssetDeps, { destination, fallbackDateTime, mediaContent, timeOffset, }: DownloadAssetOptions): Promise<{
15
16
  bytes: number;
17
+ modifiedTime: string;
16
18
  sha256: string;
17
19
  }>;
18
20
  export {};
@@ -1 +1 @@
1
- {"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../../src/downloader/asset/download.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,UAAU,iBAAiB;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,aAAa,CAAC;CAC1B;AAED,UAAU,oBAAoB;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,YAAY,CAAC;CAC5B;AAkBD,wBAAsB,aAAa,CACjC,EAAE,OAAY,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,iBAAiB,EACtD,EAAE,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,EAAE,oBAAoB;;;GAoDtE"}
1
+ {"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../../src/downloader/asset/download.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAO9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,UAAU,iBAAiB;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,aAAa,CAAC;CAC1B;AAED,UAAU,oBAAoB;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,YAAY,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;CACpB;AAkBD,wBAAsB,aAAa,CACjC,EAAE,OAAY,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,iBAAiB,EACtD,EACE,WAAW,EACX,gBAAgB,EAChB,YAAY,EACZ,UAAU,GACX,EAAE,oBAAoB;;;;GAwDxB"}
@@ -1,9 +1,8 @@
1
- import { createHash } from "node:crypto";
2
- import { createReadStream, createWriteStream } from "node:fs";
3
- import { mkdir, rename, utimes } from "node:fs/promises";
1
+ import { createWriteStream } from "node:fs";
2
+ import { mkdir, rename, stat, utimes } from "node:fs/promises";
4
3
  import path from "node:path";
5
4
  import { pipeline } from "node:stream/promises";
6
- import { exists, filesize } from "../fs/filesystem.js";
5
+ import { exists, filesize, formatFileTimestamp, hashFile, } from "../fs/filesystem.js";
7
6
  class AssetDownloadError extends Error {
8
7
  constructor(response, url, body) {
9
8
  super(`Asset download failed: ${response.status} ${url}`);
@@ -14,14 +13,14 @@ class AssetDownloadError extends Error {
14
13
  this.url = url;
15
14
  }
16
15
  }
17
- export async function downloadAsset({ headers = {}, logger, transport }, { destination, fallbackDateTime, mediaContent }) {
16
+ export async function downloadAsset({ headers = {}, logger, transport }, { destination, fallbackDateTime, mediaContent, timeOffset, }) {
18
17
  if (await exists(destination)) {
19
18
  logger.debug(`Asset ${mediaContent.id} already exists at ${destination}`);
20
- const [{ size: bytes }, sha256] = await Promise.all([
21
- filesize(destination),
19
+ const [{ mtime, size: bytes }, sha256] = await Promise.all([
20
+ stat(destination),
22
21
  hashFile(destination),
23
22
  ]);
24
- return { bytes, sha256 };
23
+ return { bytes, modifiedTime: formatFileTimestamp(mtime), sha256 };
25
24
  }
26
25
  logger.debug(`Downloading ${mediaContent.type} asset ${mediaContent.id} to ${destination}`);
27
26
  await mkdir(path.dirname(destination), { recursive: true });
@@ -38,20 +37,18 @@ export async function downloadAsset({ headers = {}, logger, transport }, { desti
38
37
  }
39
38
  await write(tempFilePath, response, downloadedBytes);
40
39
  const modified = response.headers.get("Last-Modified");
41
- const timestamp = modified ? new Date(modified) : new Date(fallbackDateTime);
40
+ const timestamp = offsetTimestamp(modified ? new Date(modified) : new Date(fallbackDateTime), timeOffset);
42
41
  await utimes(tempFilePath, timestamp, timestamp);
43
42
  const [{ size: bytes }, sha256] = await Promise.all([
44
43
  filesize(tempFilePath),
45
44
  hashFile(tempFilePath),
46
45
  ]);
47
46
  await rename(tempFilePath, destination);
48
- return { bytes, sha256 };
47
+ const { mtime } = await stat(destination);
48
+ return { bytes, modifiedTime: formatFileTimestamp(mtime), sha256 };
49
49
  }
50
- async function hashFile(filePath) {
51
- const hash = createHash("sha256");
52
- for await (const chunk of createReadStream(filePath))
53
- hash.update(chunk);
54
- return hash.digest("hex");
50
+ function offsetTimestamp(timestamp, timeOffset) {
51
+ return new Date(timestamp.getTime() + timeOffset * 1000);
55
52
  }
56
53
  async function write(path, response, partialBytes) {
57
54
  return pipeline(response.body ?? "", createWriteStream(path, {
@@ -1 +1 @@
1
- {"version":3,"file":"download.js","sourceRoot":"","sources":["../../../../src/downloader/asset/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC9D,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAKhD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAevD,MAAM,kBAAmB,SAAQ,KAAK;IAMpC,YAAmB,QAAkB,EAAE,GAAW,EAAE,IAAa;QAC/D,KAAK,CAAC,0BAA0B,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;QACtC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,EAAE,OAAO,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,EAAqB,EACtD,EAAE,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAwB;IAErE,IAAI,MAAM,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,EAAE,sBAAsB,WAAW,EAAE,CAAC,CAAC;QAC1E,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAClD,QAAQ,CAAC,WAAW,CAAC;YACrB,QAAQ,CAAC,WAAW,CAAC;SACtB,CAAC,CAAC;QAEH,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAC3B,CAAC;IAED,MAAM,CAAC,KAAK,CACV,eAAe,YAAY,CAAC,IAAI,UAAU,YAAY,CAAC,EAAE,OAAO,WAAW,EAAE,CAC9E,CAAC;IACF,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,YAAY,GAAG,WAAW,GAAG,OAAO,CAAC;IAE3C,MAAM,cAAc,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IACtC,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC/D,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,KAAK,CACV,wBAAwB,YAAY,CAAC,IAAI,UAAU,YAAY,CAAC,EAAE,OAAO,YAAY,cAAc,eAAe,EAAE,CACrH,CAAC;QACF,cAAc,CAAC,KAAK,GAAG,SAAS,eAAe,GAAG,CAAC;IACrD,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,KAAK,CACpC,IAAI,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAC3D,CAAC;IACF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,kBAAkB,CAC1B,QAAQ,EACR,YAAY,CAAC,GAAG,EAChB,MAAM,QAAQ,CAAC,IAAI,EAAE,CACtB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,CAAC,YAAY,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;IAErD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC7E,MAAM,MAAM,CAAC,YAAY,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAEjD,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAClD,QAAQ,CAAC,YAAY,CAAC;QACtB,QAAQ,CAAC,YAAY,CAAC;KACvB,CAAC,CAAC;IAEH,MAAM,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IAExC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AAC3B,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,QAAgB;IACtC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAElC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,gBAAgB,CAAC,QAAQ,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,KAAe,CAAC,CAAC;IAE/B,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,IAAY,EAAE,QAAkB,EAAE,YAAoB;IACzE,OAAO,QAAQ,CACb,QAAQ,CAAC,IAAI,IAAI,EAAE,EACnB,iBAAiB,CAAC,IAAI,EAAE;QACtB,KAAK,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;KAC/D,CAAC,CACH,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"download.js","sourceRoot":"","sources":["../../../../src/downloader/asset/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAKhD,OAAO,EACL,MAAM,EACN,QAAQ,EACR,mBAAmB,EACnB,QAAQ,GACT,MAAM,qBAAqB,CAAC;AAgB7B,MAAM,kBAAmB,SAAQ,KAAK;IAMpC,YAAmB,QAAkB,EAAE,GAAW,EAAE,IAAa;QAC/D,KAAK,CAAC,0BAA0B,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;QACtC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,EAAE,OAAO,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,EAAqB,EACtD,EACE,WAAW,EACX,gBAAgB,EAChB,YAAY,EACZ,UAAU,GACW;IAEvB,IAAI,MAAM,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,EAAE,sBAAsB,WAAW,EAAE,CAAC,CAAC;QAC1E,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACzD,IAAI,CAAC,WAAW,CAAC;YACjB,QAAQ,CAAC,WAAW,CAAC;SACtB,CAAC,CAAC;QAEH,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,mBAAmB,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACrE,CAAC;IAED,MAAM,CAAC,KAAK,CACV,eAAe,YAAY,CAAC,IAAI,UAAU,YAAY,CAAC,EAAE,OAAO,WAAW,EAAE,CAC9E,CAAC;IACF,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,YAAY,GAAG,WAAW,GAAG,OAAO,CAAC;IAE3C,MAAM,cAAc,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IACtC,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC/D,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,KAAK,CACV,wBAAwB,YAAY,CAAC,IAAI,UAAU,YAAY,CAAC,EAAE,OAAO,YAAY,cAAc,eAAe,EAAE,CACrH,CAAC;QACF,cAAc,CAAC,KAAK,GAAG,SAAS,eAAe,GAAG,CAAC;IACrD,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,KAAK,CACpC,IAAI,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAC3D,CAAC;IACF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,kBAAkB,CAC1B,QAAQ,EACR,YAAY,CAAC,GAAG,EAChB,MAAM,QAAQ,CAAC,IAAI,EAAE,CACtB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,CAAC,YAAY,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;IAErD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,eAAe,CAC/B,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC,EAC1D,UAAU,CACX,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAEjD,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAClD,QAAQ,CAAC,YAAY,CAAC;QACtB,QAAQ,CAAC,YAAY,CAAC;KACvB,CAAC,CAAC;IAEH,MAAM,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IACxC,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC;IAE1C,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,mBAAmB,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;AACrE,CAAC;AAED,SAAS,eAAe,CAAC,SAAe,EAAE,UAAkB;IAC1D,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,UAAU,GAAG,IAAK,CAAC,CAAC;AAC5D,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,IAAY,EAAE,QAAkB,EAAE,YAAoB;IACzE,OAAO,QAAQ,CACb,QAAQ,CAAC,IAAI,IAAI,EAAE,EACnB,iBAAiB,CAAC,IAAI,EAAE;QACtB,KAAK,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;KAC/D,CAAC,CACH,CAAC;AACJ,CAAC"}
@@ -1,5 +1,5 @@
1
1
  import type { Logger } from "pino";
2
- export declare const DOWNLOAD_HELP = "Usage: fanbox-dl download [options]\n\nDownload FANBOX posts for selected creators.\n\nSelectors:\n --creator <id> Add a creator ID. Can be repeated.\n --following Add all followed creators.\n --supporting Add all supporting creators.\n --ignore-creator <id> Exclude a creator ID. Can be repeated.\n\nAuth:\n --cookie <value> Raw session ID or FANBOXSESSID=... cookie.\n --cookie-file <path> Read raw cookie or Netscape cookies.txt.\n --user-agent <value> Send the User-Agent from your logged-in browser.\n FANBOX_SESSION_ID Environment fallback.\n FANBOX_USER_AGENT User-Agent environment fallback.\n\nDownload:\n --output <path> Output directory. Default: fanbox-downloads.\n --flat-posts Store post files directly under each creator.\n --max-filename-bytes <n> Max filename bytes including .part temp suffix. Default: 256.\n --flat-parent-min-bytes <n>\n Min optional parent/post name bytes to preserve in flat-posts filenames. Default: 35.\n\nRequests:\n --concurrency <n> Concurrent requests. Default: 10.\n --http2-session-multiplier <n>\n HTTP/2 sessions per origin multiplier based on concurrency. Default: 10.\n --request-interval-ms <n> Delay between request starts. Default: 500.\n --rate-limit-pause-ms <n> Force overwrite pause ms when 429.\n --max-retries <n> Retry attempts. Default: 3.\n\nOutput:\n --log-level <level> fatal|error|warn|info|debug|trace|silent. Default: info.\n --help Show this help.\n";
2
+ export declare const DOWNLOAD_HELP = "Usage: fanbox-dl download [options]\n\nDownload FANBOX posts for selected creators.\n\nSelectors:\n --creator <id> Add a creator ID. Can be repeated.\n --following Add all followed creators.\n --supporting Add all supporting creators.\n --ignore-creator <id> Exclude a creator ID. Can be repeated.\n\nAuth:\n --cookie <value> Raw session ID or FANBOXSESSID=... cookie.\n --cookie-file <path> Read raw cookie or Netscape cookies.txt.\n --user-agent <value> Send the User-Agent from your logged-in browser.\n FANBOX_SESSION_ID Environment fallback.\n\nDownload:\n --output <path> Output directory. Default: fanbox-downloads.\n --flat-posts Store post files directly under each creator.\n --max-filename-bytes <n> Max filename bytes including .part temp suffix. Default: 256.\n --flat-parent-min-bytes <n>\n Min optional parent/post name bytes to preserve in flat-posts filenames. Default: 35.\n --verify Verify skipped complete posts against local files.\n\nRequests:\n --concurrency <n> Concurrent requests. Default: 10.\n --http2-session-multiplier <n>\n HTTP/2 sessions per origin multiplier based on concurrency. Default: 10.\n --request-interval-ms <n> Delay between request starts. Default: 500.\n --rate-limit-pause-ms <n> Force overwrite pause ms when 429.\n --max-retries <n> Retry attempts. Default: 3.\n\nOutput:\n --log-level <level> fatal|error|warn|info|debug|trace|silent. Default: info.\n --help Show this help.\n";
3
3
  export interface DownloadOptions {
4
4
  concurrency: number;
5
5
  cookie?: string;
@@ -16,6 +16,7 @@ export interface DownloadOptions {
16
16
  requestIntervalMs: number;
17
17
  supporting: boolean;
18
18
  userAgent?: string;
19
+ verify: boolean;
19
20
  }
20
21
  interface ParseDownloadOptionsDeps {
21
22
  logger: Logger;
@@ -1 +1 @@
1
- {"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../../../src/downloader/cli/options.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAKnC,eAAO,MAAM,aAAa,umDAmCzB,CAAC;AAEF,MAAM,WAAW,eAAe;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,OAAO,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,wBAAwB;IAChC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,oBAAoB,CAClC,EAAE,MAAM,EAAE,EAAE,wBAAwB,EACpC,IAAI,EAAE,MAAM,EAAE,GACb,eAAe,CA2CjB"}
1
+ {"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../../../src/downloader/cli/options.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAKnC,eAAO,MAAM,aAAa,ynDAmCzB,CAAC;AAEF,MAAM,WAAW,eAAe;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,OAAO,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,UAAU,wBAAwB;IAChC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,oBAAoB,CAClC,EAAE,MAAM,EAAE,EAAE,wBAAwB,EACpC,IAAI,EAAE,MAAM,EAAE,GACb,eAAe,CA4CjB"}
@@ -17,7 +17,6 @@ Auth:
17
17
  --cookie-file <path> Read raw cookie or Netscape cookies.txt.
18
18
  --user-agent <value> Send the User-Agent from your logged-in browser.
19
19
  FANBOX_SESSION_ID Environment fallback.
20
- FANBOX_USER_AGENT User-Agent environment fallback.
21
20
 
22
21
  Download:
23
22
  --output <path> Output directory. Default: fanbox-downloads.
@@ -25,6 +24,7 @@ Download:
25
24
  --max-filename-bytes <n> Max filename bytes including .part temp suffix. Default: 256.
26
25
  --flat-parent-min-bytes <n>
27
26
  Min optional parent/post name bytes to preserve in flat-posts filenames. Default: 35.
27
+ --verify Verify skipped complete posts against local files.
28
28
 
29
29
  Requests:
30
30
  --concurrency <n> Concurrent requests. Default: 10.
@@ -63,6 +63,7 @@ export function parseDownloadOptions({ logger }, args) {
63
63
  requestIntervalMs: parseNonNegativeInteger("request-interval-ms", values["request-interval-ms"]),
64
64
  supporting: values.supporting,
65
65
  userAgent: values["user-agent"],
66
+ verify: values.verify,
66
67
  };
67
68
  }
68
69
  function parseCookie({ logger }, values) {
@@ -106,6 +107,7 @@ function parseDownloadArgs(args) {
106
107
  "request-interval-ms": { default: "500", type: "string" },
107
108
  supporting: { default: false, type: "boolean" },
108
109
  "user-agent": { type: "string" },
110
+ verify: { default: false, type: "boolean" },
109
111
  },
110
112
  strict: true,
111
113
  });
@@ -1 +1 @@
1
- {"version":3,"file":"options.js","sourceRoot":"","sources":["../../../../src/downloader/cli/options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAItC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C,MAAM,CAAC,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmC5B,CAAC;AAwBF,MAAM,UAAU,oBAAoB,CAClC,EAAE,MAAM,EAA4B,EACpC,IAAc;IAEd,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,EAAE,0BAA0B,CAAC,CAAC;IACnD,MAAM,EAAE,MAAM,EAAE,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IACxC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,UAAU;QACpE,MAAM,IAAI,aAAa,CAAC,2CAA2C,CAAC,CAAC;IAEvE,MAAM,MAAM,GAAG,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,CAAC;IAE/C,OAAO;QACL,WAAW,EAAE,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,WAAW,CAAC;QACpE,MAAM;QACN,UAAU;QACV,kBAAkB,EAAE,oBAAoB,CACtC,uBAAuB,EACvB,MAAM,CAAC,uBAAuB,CAAC,CAChC;QACD,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC;QAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,sBAAsB,EAAE,oBAAoB,CAC1C,0BAA0B,EAC1B,MAAM,CAAC,0BAA0B,CAAC,CACnC;QACD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;QAChD,gBAAgB,EAAE,oBAAoB,CACpC,oBAAoB,EACpB,MAAM,CAAC,oBAAoB,CAAC,CAC7B;QACD,UAAU,EAAE,uBAAuB,CAAC,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QACzE,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,gBAAgB,EAAE,MAAM,CAAC,qBAAqB,CAAC;YAC7C,CAAC,CAAC,uBAAuB,CACrB,qBAAqB,EACrB,MAAM,CAAC,qBAAqB,CAAC,CAC9B;YACH,CAAC,CAAC,SAAS;QACb,iBAAiB,EAAE,uBAAuB,CACxC,qBAAqB,EACrB,MAAM,CAAC,qBAAqB,CAAC,CAC9B;QACD,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC;KAChC,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAClB,EAAE,MAAM,EAA4B,EACpC,MAAsD;IAEtD,MAAM,UAAU,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,EAAE,8BAA8B,CAAC,CAAC;QACxE,OAAO,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,EAAE,wBAAwB,CAAC,CAAC;QACvD,OAAO,eAAe,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;QAClC,MAAM,CAAC,KAAK,CACV,EAAE,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,EACpD,+BAA+B,CAChC,CAAC;QACF,OAAO,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACnC,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAc;IACvC,IAAI,CAAC;QACH,OAAO,SAAS,CAAC;YACf,gBAAgB,EAAE,KAAK;YACvB,IAAI;YACJ,OAAO,EAAE;gBACP,WAAW,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC9C,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC1B,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACjC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC3C,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC9C,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC1D,YAAY,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBACjD,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC9C,0BAA0B,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC7D,gBAAgB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACpD,WAAW,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAChD,oBAAoB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACxD,aAAa,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC/C,MAAM,EAAE,EAAE,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACvD,qBAAqB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzC,qBAAqB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzD,UAAU,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC/C,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aACjC;YACD,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,aAAa,CAAE,KAAe,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,IAAY,EAAE,KAAa;IAC1D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,aAAa,CAAC,GAAG,IAAI,iCAAiC,CAAC,CAAC;IACpE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY,EAAE,KAAa;IACvD,MAAM,MAAM,GAAG,uBAAuB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACpD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,MAAM,IAAI,aAAa,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
1
+ {"version":3,"file":"options.js","sourceRoot":"","sources":["../../../../src/downloader/cli/options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAItC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C,MAAM,CAAC,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmC5B,CAAC;AAyBF,MAAM,UAAU,oBAAoB,CAClC,EAAE,MAAM,EAA4B,EACpC,IAAc;IAEd,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,EAAE,0BAA0B,CAAC,CAAC;IACnD,MAAM,EAAE,MAAM,EAAE,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IACxC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,UAAU;QACpE,MAAM,IAAI,aAAa,CAAC,2CAA2C,CAAC,CAAC;IAEvE,MAAM,MAAM,GAAG,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,CAAC;IAE/C,OAAO;QACL,WAAW,EAAE,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,WAAW,CAAC;QACpE,MAAM;QACN,UAAU;QACV,kBAAkB,EAAE,oBAAoB,CACtC,uBAAuB,EACvB,MAAM,CAAC,uBAAuB,CAAC,CAChC;QACD,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC;QAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,sBAAsB,EAAE,oBAAoB,CAC1C,0BAA0B,EAC1B,MAAM,CAAC,0BAA0B,CAAC,CACnC;QACD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;QAChD,gBAAgB,EAAE,oBAAoB,CACpC,oBAAoB,EACpB,MAAM,CAAC,oBAAoB,CAAC,CAC7B;QACD,UAAU,EAAE,uBAAuB,CAAC,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QACzE,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,gBAAgB,EAAE,MAAM,CAAC,qBAAqB,CAAC;YAC7C,CAAC,CAAC,uBAAuB,CACrB,qBAAqB,EACrB,MAAM,CAAC,qBAAqB,CAAC,CAC9B;YACH,CAAC,CAAC,SAAS;QACb,iBAAiB,EAAE,uBAAuB,CACxC,qBAAqB,EACrB,MAAM,CAAC,qBAAqB,CAAC,CAC9B;QACD,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;KACtB,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAClB,EAAE,MAAM,EAA4B,EACpC,MAAsD;IAEtD,MAAM,UAAU,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,EAAE,8BAA8B,CAAC,CAAC;QACxE,OAAO,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,EAAE,wBAAwB,CAAC,CAAC;QACvD,OAAO,eAAe,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;QAClC,MAAM,CAAC,KAAK,CACV,EAAE,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,EACpD,+BAA+B,CAChC,CAAC;QACF,OAAO,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACnC,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAc;IACvC,IAAI,CAAC;QACH,OAAO,SAAS,CAAC;YACf,gBAAgB,EAAE,KAAK;YACvB,IAAI;YACJ,OAAO,EAAE;gBACP,WAAW,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC9C,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC1B,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACjC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC3C,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC9C,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC1D,YAAY,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBACjD,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC9C,0BAA0B,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC7D,gBAAgB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACpD,WAAW,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAChD,oBAAoB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACxD,aAAa,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC/C,MAAM,EAAE,EAAE,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACvD,qBAAqB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzC,qBAAqB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzD,UAAU,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC/C,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAChC,MAAM,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;aAC5C;YACD,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,aAAa,CAAE,KAAe,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,IAAY,EAAE,KAAa;IAC1D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,aAAa,CAAC,GAAG,IAAI,iCAAiC,CAAC,CAAC;IACpE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY,EAAE,KAAa;IACvD,MAAM,MAAM,GAAG,uBAAuB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACpD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,MAAM,IAAI,aAAa,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"discover-posts.d.ts","sourceRoot":"","sources":["../../../../src/downloader/creator/discover-posts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAEV,WAAW,EACZ,MAAM,uBAAuB,CAAC;AAE/B,UAAU,gCAAgC;IACxC,MAAM,EAAE,YAAY,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,2BAA2B;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,gCAAgC,EACpD,EACE,SAAS,EACT,OAAO,EACP,sBAAsB,EACtB,KAAW,GACZ,EAAE,2BAA2B,GAC7B,OAAO,CAAC,WAAW,EAAE,CAAC,CA+BxB"}
1
+ {"version":3,"file":"discover-posts.d.ts","sourceRoot":"","sources":["../../../../src/downloader/creator/discover-posts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAEV,WAAW,EACZ,MAAM,uBAAuB,CAAC;AAE/B,UAAU,gCAAgC;IACxC,MAAM,EAAE,YAAY,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,2BAA2B;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,gCAAgC,EACpD,EACE,SAAS,EACT,OAAO,EACP,sBAAsB,EACtB,KAAW,GACZ,EAAE,2BAA2B,GAC7B,OAAO,CAAC,WAAW,EAAE,CAAC,CA6BxB"}
@@ -11,11 +11,13 @@ export async function discoverCreatorPosts({ client, logger }, { creatorId, firs
11
11
  logger.debug(`Discovered ${posts.length} posts for creator ${creatorId}`);
12
12
  if (posts.length < limit)
13
13
  return posts;
14
- const last = posts.at(-1);
14
+ const lastPost = posts.at(-1);
15
+ if (!lastPost)
16
+ return posts;
15
17
  const remains = await discoverCreatorPosts({ client, logger }, {
16
18
  creatorId,
17
- firstId: last.id,
18
- firstPublishedDatetime: last.publishedDatetime,
19
+ firstId: lastPost.id,
20
+ firstPublishedDatetime: lastPost.publishedDatetime,
19
21
  limit,
20
22
  });
21
23
  return [...posts, ...remains.slice(1)];
@@ -1 +1 @@
1
- {"version":3,"file":"discover-posts.js","sourceRoot":"","sources":["../../../../src/downloader/creator/discover-posts.ts"],"names":[],"mappings":"AAoBA,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAoC,EACpD,EACE,SAAS,EACT,OAAO,EACP,sBAAsB,EACtB,KAAK,GAAG,GAAG,GACiB;IAE9B,MAAM,CAAC,KAAK,CACV,mBAAmB,KAAK,eAAe,OAAO,IAAI,OAAO,gBAAgB,SAAS,EAAE,CACrF,CAAC;IACF,MAAM,uBAAuB,GAAG;QAC9B,SAAS;QACT,OAAO;QACP,sBAAsB;QACtB,KAAK;QACL,IAAI,EAAE,QAAQ;KACkB,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;IACrE,MAAM,CAAC,KAAK,CAAC,cAAc,KAAK,CAAC,MAAM,sBAAsB,SAAS,EAAE,CAAC,CAAC;IAC1E,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK;QAAE,OAAO,KAAK,CAAC;IAIvC,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC;IAG3B,MAAM,OAAO,GAAG,MAAM,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAE,EAClB;QACE,SAAS;QACT,OAAO,EAAE,IAAI,CAAC,EAAE;QAChB,sBAAsB,EAAE,IAAI,CAAC,iBAAiB;QAC9C,KAAK;KACN,CACF,CAAC;IAEF,OAAO,CAAC,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC"}
1
+ {"version":3,"file":"discover-posts.js","sourceRoot":"","sources":["../../../../src/downloader/creator/discover-posts.ts"],"names":[],"mappings":"AAoBA,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAoC,EACpD,EACE,SAAS,EACT,OAAO,EACP,sBAAsB,EACtB,KAAK,GAAG,GAAG,GACiB;IAE9B,MAAM,CAAC,KAAK,CACV,mBAAmB,KAAK,eAAe,OAAO,IAAI,OAAO,gBAAgB,SAAS,EAAE,CACrF,CAAC;IACF,MAAM,uBAAuB,GAAG;QAC9B,SAAS;QACT,OAAO;QACP,sBAAsB;QACtB,KAAK;QACL,IAAI,EAAE,QAAQ;KACkB,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;IACrE,MAAM,CAAC,KAAK,CAAC,cAAc,KAAK,CAAC,MAAM,sBAAsB,SAAS,EAAE,CAAC,CAAC;IAC1E,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK;QAAE,OAAO,KAAK,CAAC;IAEvC,MAAM,QAAQ,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAE5B,MAAM,OAAO,GAAG,MAAM,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAE,EAClB;QACE,SAAS;QACT,OAAO,EAAE,QAAQ,CAAC,EAAE;QACpB,sBAAsB,EAAE,QAAQ,CAAC,iBAAiB;QAClD,KAAK;KACN,CACF,CAAC;IAEF,OAAO,CAAC,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC"}
@@ -10,7 +10,8 @@ interface SyncCreatorDeps {
10
10
  manifest: CreatorManifest;
11
11
  pathManager: PathManager;
12
12
  transport: HttpTransport;
13
+ verify?: boolean;
13
14
  }
14
- export declare function syncCreator({ client, headers, logger, manifest, pathManager, transport, }: SyncCreatorDeps): Promise<void>;
15
+ export declare function syncCreator({ client, headers, logger, manifest, pathManager, transport, verify, }: SyncCreatorDeps): Promise<void>;
15
16
  export {};
16
17
  //# sourceMappingURL=sync.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../../../src/downloader/creator/sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAoB,MAAM,wBAAwB,CAAC;AAIhF,UAAU,eAAe;IACvB,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,eAAe,CAAC;IAC1B,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,aAAa,CAAC;CAC1B;AAED,wBAAsB,WAAW,CAAC,EAChC,MAAM,EACN,OAAO,EACP,MAAM,EACN,QAAQ,EACR,WAAW,EACX,SAAS,GACV,EAAE,eAAe,iBA6EjB"}
1
+ {"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../../../src/downloader/creator/sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAoB,MAAM,wBAAwB,CAAC;AAUhF,UAAU,eAAe;IACvB,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,eAAe,CAAC;IAC1B,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,aAAa,CAAC;IACzB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,wBAAsB,WAAW,CAAC,EAChC,MAAM,EACN,OAAO,EACP,MAAM,EACN,QAAQ,EACR,WAAW,EACX,SAAS,EACT,MAAc,GACf,EAAE,eAAe,iBAoEjB"}
@@ -1,14 +1,15 @@
1
1
  import { preSyncPostCheck, syncPost } from "../post/sync.js";
2
2
  import { discoverCreatorPosts } from "./discover-posts.js";
3
- export async function syncCreator({ client, headers, logger, manifest, pathManager, transport, }) {
3
+ export async function syncCreator({ client, headers, logger, manifest, pathManager, transport, verify = false, }) {
4
4
  const postSummaries = await discoverCreatorPosts({ client, logger }, { creatorId: manifest.creatorId });
5
5
  logger.info(`Discovered total ${postSummaries.length} posts for creator ${manifest.creatorId}`);
6
6
  const processingPosts = [];
7
- let index = 0;
8
- for (const postSummary of postSummaries) {
9
- logger.debug(`Initializing ${++index}/${postSummaries.length} post ${postSummary.id} for creator ${manifest.creatorId}`);
10
- const preCheckResult = preSyncPostCheck({ logger, manifest }, postSummary);
11
- if (preCheckResult.status !== "pending") {
7
+ for (const [index, postSummary] of postSummaries.entries()) {
8
+ const postIndex = index + 1;
9
+ logger.debug(`Initializing ${postIndex}/${postSummaries.length} post ${postSummary.id} for creator ${manifest.creatorId}`);
10
+ const preCheckResult = await preSyncPostCheck({ logger, manifest, verify }, postSummary);
11
+ if (preCheckResult.status === "complete" ||
12
+ preCheckResult.status === "skipped") {
12
13
  manifest.posts[postSummary.id] = preCheckResult;
13
14
  continue;
14
15
  }
@@ -16,16 +17,8 @@ export async function syncCreator({ client, headers, logger, manifest, pathManag
16
17
  const post = await client
17
18
  .getPost({ postId: postSummary.id })
18
19
  .catch(async (err) => {
19
- manifest.posts[postSummary.id] = {
20
- assets: {},
21
- error: String(err),
22
- id: postSummary.id,
23
- restricted: postSummary.isRestricted,
24
- status: "failed",
25
- updatedDatetime: postSummary.updatedDatetime,
26
- };
27
- logger.error({ err }, `Error occurred while fetch post manifest of ${index}/${postSummaries.length} post ${postSummary.id}, skipping.`);
28
- await manifest.save();
20
+ logger.error({ err }, `Error occurred while fetch post manifest of ${postIndex}/${postSummaries.length} post ${postSummary.id}, skipping.`);
21
+ await saveFailedPostManifest(manifest, postSummary, err);
29
22
  });
30
23
  if (!post)
31
24
  continue;
@@ -35,20 +28,26 @@ export async function syncCreator({ client, headers, logger, manifest, pathManag
35
28
  return manifest.save();
36
29
  })
37
30
  .catch((err) => {
38
- manifest.posts[postSummary.id] = {
39
- assets: {},
40
- error: String(err),
41
- id: postSummary.id,
42
- restricted: postSummary.isRestricted,
43
- status: "failed",
44
- updatedDatetime: postSummary.updatedDatetime,
45
- };
46
- logger.error({ err }, `Error occurred while syncing ${index}/${postSummaries.length} post ${postSummary.id}, skipping.`);
47
- return manifest.save();
31
+ logger.error({ err }, `Error occurred while syncing ${postIndex}/${postSummaries.length} post ${postSummary.id}, skipping.`);
32
+ return saveFailedPostManifest(manifest, postSummary, err);
48
33
  });
49
34
  processingPosts.push(syncPostPromise);
50
35
  }
51
36
  logger.info(`All ${postSummaries.length} posts of creator ${manifest.creatorId} have been initialized for sync, waiting for all to complete...`);
52
37
  await Promise.all(processingPosts);
53
38
  }
39
+ function failedPostManifestData(postSummary, err) {
40
+ return {
41
+ assets: {},
42
+ error: String(err),
43
+ id: postSummary.id,
44
+ restricted: postSummary.isRestricted,
45
+ status: "failed",
46
+ updatedDatetime: postSummary.updatedDatetime,
47
+ };
48
+ }
49
+ async function saveFailedPostManifest(manifest, postSummary, err) {
50
+ manifest.posts[postSummary.id] = failedPostManifestData(postSummary, err);
51
+ await manifest.save();
52
+ }
54
53
  //# sourceMappingURL=sync.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"sync.js","sourceRoot":"","sources":["../../../../src/downloader/creator/sync.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAW3D,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAChC,MAAM,EACN,OAAO,EACP,MAAM,EACN,QAAQ,EACR,WAAW,EACX,SAAS,GACO;IAChB,MAAM,aAAa,GAAG,MAAM,oBAAoB,CAC9C,EAAE,MAAM,EAAE,MAAM,EAAE,EAClB,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,CAClC,CAAC;IACF,MAAM,CAAC,IAAI,CACT,oBAAoB,aAAa,CAAC,MAAM,sBAAsB,QAAQ,CAAC,SAAS,EAAE,CACnF,CAAC;IAEF,MAAM,eAAe,GAAoB,EAAE,CAAC;IAE5C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,WAAW,IAAI,aAAa,EAAE,CAAC;QACxC,MAAM,CAAC,KAAK,CACV,gBAAgB,EAAE,KAAK,IAAI,aAAa,CAAC,MAAM,SAAS,WAAW,CAAC,EAAE,gBAAgB,QAAQ,CAAC,SAAS,EAAE,CAC3G,CAAC;QACF,MAAM,cAAc,GAAG,gBAAgB,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,WAAW,CAAC,CAAC;QAC3E,IAAI,cAAc,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACxC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC;YAChD,SAAS;QACX,CAAC;QAKD,MAAM,eAAe,GAAG,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtD,MAAM,IAAI,GAAG,MAAM,MAAM;aACtB,OAAO,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC;aACnC,KAAK,CAAC,KAAK,EAAE,GAAY,EAAE,EAAE;YAC5B,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,GAAG;gBAC/B,MAAM,EAAE,EAAE;gBACV,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;gBAClB,EAAE,EAAE,WAAW,CAAC,EAAE;gBAClB,UAAU,EAAE,WAAW,CAAC,YAAY;gBACpC,MAAM,EAAE,QAAQ;gBAChB,eAAe,EAAE,WAAW,CAAC,eAAe;aAClB,CAAC;YAC7B,MAAM,CAAC,KAAK,CACV,EAAE,GAAG,EAAE,EACP,+CAA+C,KAAK,IAAI,aAAa,CAAC,MAAM,SAAS,WAAW,CAAC,EAAE,aAAa,CACjH,CAAC;YACF,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACxB,CAAC,CAAC,CAAC;QACL,IAAI,CAAC,IAAI;YAAE,SAAS;QAEpB,MAAM,eAAe,GAAG,QAAQ,CAC9B,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,eAAe,EAAE,SAAS,EAAE,EAC5D,IAAI,CACL;aACE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YACrB,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC;YACxC,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;QACzB,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YACtB,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,GAAG;gBAC/B,MAAM,EAAE,EAAE;gBACV,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;gBAClB,EAAE,EAAE,WAAW,CAAC,EAAE;gBAClB,UAAU,EAAE,WAAW,CAAC,YAAY;gBACpC,MAAM,EAAE,QAAQ;gBAChB,eAAe,EAAE,WAAW,CAAC,eAAe;aAClB,CAAC;YAC7B,MAAM,CAAC,KAAK,CACV,EAAE,GAAG,EAAE,EACP,gCAAgC,KAAK,IAAI,aAAa,CAAC,MAAM,SAAS,WAAW,CAAC,EAAE,aAAa,CAClG,CAAC;YACF,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;QACzB,CAAC,CAAC,CAAC;QAEL,eAAe,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,CAAC,IAAI,CACT,OAAO,aAAa,CAAC,MAAM,qBAAqB,QAAQ,CAAC,SAAS,iEAAiE,CACpI,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;AACrC,CAAC"}
1
+ {"version":3,"file":"sync.js","sourceRoot":"","sources":["../../../../src/downloader/creator/sync.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAkB3D,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAChC,MAAM,EACN,OAAO,EACP,MAAM,EACN,QAAQ,EACR,WAAW,EACX,SAAS,EACT,MAAM,GAAG,KAAK,GACE;IAChB,MAAM,aAAa,GAAG,MAAM,oBAAoB,CAC9C,EAAE,MAAM,EAAE,MAAM,EAAE,EAClB,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,CAClC,CAAC;IACF,MAAM,CAAC,IAAI,CACT,oBAAoB,aAAa,CAAC,MAAM,sBAAsB,QAAQ,CAAC,SAAS,EAAE,CACnF,CAAC;IAEF,MAAM,eAAe,GAAoB,EAAE,CAAC;IAE5C,KAAK,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3D,MAAM,SAAS,GAAG,KAAK,GAAG,CAAC,CAAC;QAC5B,MAAM,CAAC,KAAK,CACV,gBAAgB,SAAS,IAAI,aAAa,CAAC,MAAM,SAAS,WAAW,CAAC,EAAE,gBAAgB,QAAQ,CAAC,SAAS,EAAE,CAC7G,CAAC;QACF,MAAM,cAAc,GAAG,MAAM,gBAAgB,CAC3C,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAC5B,WAAW,CACZ,CAAC;QAEF,IACE,cAAc,CAAC,MAAM,KAAK,UAAU;YACpC,cAAc,CAAC,MAAM,KAAK,SAAS,EACnC,CAAC;YACD,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC;YAChD,SAAS;QACX,CAAC;QAKD,MAAM,eAAe,GAAG,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtD,MAAM,IAAI,GAAG,MAAM,MAAM;aACtB,OAAO,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC;aACnC,KAAK,CAAC,KAAK,EAAE,GAAY,EAAE,EAAE;YAC5B,MAAM,CAAC,KAAK,CACV,EAAE,GAAG,EAAE,EACP,+CAA+C,SAAS,IAAI,aAAa,CAAC,MAAM,SAAS,WAAW,CAAC,EAAE,aAAa,CACrH,CAAC;YACF,MAAM,sBAAsB,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;QACL,IAAI,CAAC,IAAI;YAAE,SAAS;QAEpB,MAAM,eAAe,GAAG,QAAQ,CAC9B,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,eAAe,EAAE,SAAS,EAAE,EAC5D,IAAI,CACL;aACE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YACrB,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC;YACxC,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;QACzB,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YACtB,MAAM,CAAC,KAAK,CACV,EAAE,GAAG,EAAE,EACP,gCAAgC,SAAS,IAAI,aAAa,CAAC,MAAM,SAAS,WAAW,CAAC,EAAE,aAAa,CACtG,CAAC;YACF,OAAO,sBAAsB,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEL,eAAe,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,CAAC,IAAI,CACT,OAAO,aAAa,CAAC,MAAM,qBAAqB,QAAQ,CAAC,SAAS,iEAAiE,CACpI,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,sBAAsB,CAC7B,WAA+B,EAC/B,GAAY;IAEZ,OAAO;QACL,MAAM,EAAE,EAAE;QACV,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;QAClB,EAAE,EAAE,WAAW,CAAC,EAAE;QAClB,UAAU,EAAE,WAAW,CAAC,YAAY;QACpC,MAAM,EAAE,QAAQ;QAChB,eAAe,EAAE,WAAW,CAAC,eAAe;KAC7C,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,QAAyB,EACzB,WAA+B,EAC/B,GAAY;IAEZ,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,GAAG,sBAAsB,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;IAC1E,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;AACxB,CAAC"}
@@ -2,5 +2,8 @@ export declare function exists(filePath: string): Promise<boolean>;
2
2
  export declare function filesize(path: string): Promise<import("fs").Stats | {
3
3
  size: number;
4
4
  }>;
5
+ export declare function formatFileTimestamp(timestamp: Date): string;
6
+ export declare function hashFile(filePath: string): Promise<string>;
5
7
  export declare function isNotFoundError(error: unknown): boolean;
8
+ export declare function normalizeFileTimestamp(timestamp: Date): Date;
6
9
  //# sourceMappingURL=filesystem.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"filesystem.d.ts","sourceRoot":"","sources":["../../../../src/downloader/fs/filesystem.ts"],"names":[],"mappings":"AAEA,wBAAsB,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAI/D;AAED,wBAAsB,QAAQ,CAAC,IAAI,EAAE,MAAM;;GAE1C;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAEvD"}
1
+ {"version":3,"file":"filesystem.d.ts","sourceRoot":"","sources":["../../../../src/downloader/fs/filesystem.ts"],"names":[],"mappings":"AAIA,wBAAsB,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAQ/D;AAED,wBAAsB,QAAQ,CAAC,IAAI,EAAE,MAAM;;GAM1C;AAED,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,IAAI,GAAG,MAAM,CAE3D;AAED,wBAAsB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAOhE;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAEvD;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,IAAI,GAAG,IAAI,CAE5D"}
@@ -1,15 +1,37 @@
1
+ import { createHash } from "node:crypto";
2
+ import { createReadStream } from "node:fs";
1
3
  import { stat } from "node:fs/promises";
2
4
  export async function exists(filePath) {
3
5
  return stat(filePath)
4
6
  .then(() => true)
5
- .catch(() => false);
7
+ .catch((error) => {
8
+ if (isNotFoundError(error))
9
+ return false;
10
+ throw error;
11
+ });
6
12
  }
7
13
  export async function filesize(path) {
8
- return stat(path).catch(() => ({ size: 0 }));
14
+ return stat(path).catch((error) => {
15
+ if (isNotFoundError(error))
16
+ return { size: 0 };
17
+ throw error;
18
+ });
19
+ }
20
+ export function formatFileTimestamp(timestamp) {
21
+ return normalizeFileTimestamp(timestamp).toISOString();
22
+ }
23
+ export async function hashFile(filePath) {
24
+ const hash = createHash("sha256");
25
+ for await (const chunk of createReadStream(filePath))
26
+ hash.update(chunk);
27
+ return hash.digest("hex");
9
28
  }
10
29
  export function isNotFoundError(error) {
11
30
  return isNodeError(error) && error.code === "ENOENT";
12
31
  }
32
+ export function normalizeFileTimestamp(timestamp) {
33
+ return new Date(Math.trunc(timestamp.getTime() / 1000) * 1000);
34
+ }
13
35
  function isNodeError(error) {
14
36
  return error instanceof Error && "code" in error;
15
37
  }
@@ -1 +1 @@
1
- {"version":3,"file":"filesystem.js","sourceRoot":"","sources":["../../../../src/downloader/fs/filesystem.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAExC,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,QAAgB;IAC3C,OAAO,IAAI,CAAC,QAAQ,CAAC;SAClB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;AACxB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAc;IAC5C,OAAO,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;AACvD,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,CAAC;AACnD,CAAC"}
1
+ {"version":3,"file":"filesystem.js","sourceRoot":"","sources":["../../../../src/downloader/fs/filesystem.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAExC,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,QAAgB;IAC3C,OAAO,IAAI,CAAC,QAAQ,CAAC;SAClB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;QACxB,IAAI,eAAe,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAEzC,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;QACzC,IAAI,eAAe,CAAC,KAAK,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QAE/C,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,SAAe;IACjD,OAAO,sBAAsB,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,QAAgB;IAC7C,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAElC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,gBAAgB,CAAC,QAAQ,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,KAAe,CAAC,CAAC;IAE/B,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAc;IAC5C,OAAO,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,SAAe;IACpD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,IAAK,CAAC,GAAG,IAAK,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,CAAC;AACnD,CAAC"}
@@ -5,5 +5,5 @@ export interface RunCliDependencies {
5
5
  logger: Logger;
6
6
  transport?: HttpTransport;
7
7
  }
8
- export declare function download({ logger, transport: customTransport }: RunCliDependencies, options: DownloadOptions): Promise<string[]>;
8
+ export declare function download({ logger, transport: customTransport }: RunCliDependencies, options: DownloadOptions): Promise<boolean>;
9
9
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/downloader/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAInC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAOxD,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,aAAa,CAAC;CAC3B;AAED,wBAAsB,QAAQ,CAC5B,EAAE,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,EAAE,kBAAkB,EAC1D,OAAO,EAAE,eAAe,qBA2EzB"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/downloader/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAInC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,aAAa,CAAC;CAC3B;AAED,wBAAsB,QAAQ,CAC5B,EAAE,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,EAAE,kBAAkB,EAC1D,OAAO,EAAE,eAAe,oBA2EzB"}
@@ -27,7 +27,6 @@ export async function download({ logger, transport: customTransport }, options)
27
27
  rootPath: options.output,
28
28
  });
29
29
  const client = new FanboxClient({ headers, transport });
30
- const failed = [];
31
30
  const creatorIds = await resolveCreatorIds({ client, logger }, options);
32
31
  const creatorManifestManager = new CreatorManifestManager({
33
32
  logger,
@@ -37,32 +36,30 @@ export async function download({ logger, transport: customTransport }, options)
37
36
  for (const creatorId of creatorIds) {
38
37
  logger.debug(`Initializing download for creator ${creatorId}`);
39
38
  const creatorManifest = await creatorManifestManager.load(creatorId);
40
- const creatorPathManager = pathManager.dir([
41
- { context: creatorId, required: true },
42
- ]);
43
39
  const syncCreatorPromise = syncCreator({
44
40
  client,
45
41
  headers,
46
42
  logger,
47
43
  manifest: creatorManifest,
48
- pathManager: creatorPathManager,
44
+ pathManager: pathManager.dir([{ context: creatorId, required: true }]),
49
45
  transport,
50
- }).catch(() => {
51
- logger.error(`Error occurred while syncing creator ${creatorId}, skipping.`);
52
- failed.push(creatorId);
46
+ verify: options.verify,
47
+ })
48
+ .then(() => {
49
+ creatorManifestManager.markSucceeded(creatorId);
50
+ })
51
+ .catch((err) => {
52
+ logger.error({ err }, `Error occurred while syncing creator ${creatorId}, skipping.`);
53
+ creatorManifestManager.markFailed(creatorId, err);
53
54
  });
54
55
  processingCreators.push(syncCreatorPromise);
55
- const success = !hasFailures(creatorManifest);
56
- if (!success)
57
- failed.push(creatorId);
58
56
  }
59
57
  logger.info(`All ${creatorIds.length} creators have been initialized for download, waiting for all to complete...`);
60
58
  await Promise.all(processingCreators);
61
59
  await creatorManifestManager.saveAll();
62
- logger.info({ failed }, `Download completed for ${creatorIds.length} creators, with ${failed.length} failures.`);
63
- return failed;
64
- }
65
- function hasFailures(manifest) {
66
- return Object.values(manifest.posts).some((post) => post?.status === "failed");
60
+ const failedCreatorIds = creatorManifestManager.getFailedCreatorIds();
61
+ logger.info({ failedCreatorIds }, `Download completed for ${creatorIds.length} creators, with ${failedCreatorIds.length} failures.`);
62
+ const hasFailedCreators = failedCreatorIds.length > 0;
63
+ return hasFailedCreators;
67
64
  }
68
65
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/downloader/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AAEzE,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AAQvE,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,EAAE,MAAM,EAAE,SAAS,EAAE,eAAe,EAAsB,EAC1D,OAAwB;IAExB,MAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,EAAE,gCAAgC,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,0BAA0B,CAAC;QACzC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IACH,MAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,EAAE,6BAA6B,CAAC,CAAC;IACzD,MAAM,SAAS,GAAG,IAAI,aAAa,CACjC,EAAE,MAAM,EAAE,EACV;QACE,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,sBAAsB,EAAE,OAAO,CAAC,sBAAsB;QACtD,UAAU,EAAE,OAAO,CAAC,iBAAiB;QACrC,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,SAAS,EAAE,eAAe;KAC3B,CACF,CAAC;IACF,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC;QAClC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,QAAQ,EAAE,OAAO,CAAC,MAAM;KACzB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;IAExD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,CAAC;IACxE,MAAM,sBAAsB,GAAG,IAAI,sBAAsB,CAAC;QACxD,MAAM;QACN,WAAW;KACZ,CAAC,CAAC;IAEH,MAAM,kBAAkB,GAAoB,EAAE,CAAC;IAE/C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,CAAC,KAAK,CAAC,qCAAqC,SAAS,EAAE,CAAC,CAAC;QAC/D,MAAM,eAAe,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACrE,MAAM,kBAAkB,GAAG,WAAW,CAAC,GAAG,CAAC;YACzC,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE;SACvC,CAAC,CAAC;QAEH,MAAM,kBAAkB,GAAG,WAAW,CAAC;YACrC,MAAM;YACN,OAAO;YACP,MAAM;YACN,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,kBAAkB;YAC/B,SAAS;SACV,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;YACZ,MAAM,CAAC,KAAK,CACV,wCAAwC,SAAS,aAAa,CAC/D,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QACH,kBAAkB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAE5C,MAAM,OAAO,GAAG,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC;QAC9C,IAAI,CAAC,OAAO;YAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,CAAC,IAAI,CACT,OAAO,UAAU,CAAC,MAAM,8EAA8E,CACvG,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IACtC,MAAM,sBAAsB,CAAC,OAAO,EAAE,CAAC;IAEvC,MAAM,CAAC,IAAI,CACT,EAAE,MAAM,EAAE,EACV,0BAA0B,UAAU,CAAC,MAAM,mBAAmB,MAAM,CAAC,MAAM,YAAY,CACxF,CAAC;IAEF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,QAAyB;IAC5C,OAAO,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CACvC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,QAAQ,CACpC,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/downloader/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AAEzE,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AAOvE,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,EAAE,MAAM,EAAE,SAAS,EAAE,eAAe,EAAsB,EAC1D,OAAwB;IAExB,MAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,EAAE,gCAAgC,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,0BAA0B,CAAC;QACzC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IACH,MAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,EAAE,6BAA6B,CAAC,CAAC;IACzD,MAAM,SAAS,GAAG,IAAI,aAAa,CACjC,EAAE,MAAM,EAAE,EACV;QACE,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,sBAAsB,EAAE,OAAO,CAAC,sBAAsB;QACtD,UAAU,EAAE,OAAO,CAAC,iBAAiB;QACrC,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,SAAS,EAAE,eAAe;KAC3B,CACF,CAAC;IACF,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC;QAClC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,QAAQ,EAAE,OAAO,CAAC,MAAM;KACzB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;IAExD,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,CAAC;IACxE,MAAM,sBAAsB,GAAG,IAAI,sBAAsB,CAAC;QACxD,MAAM;QACN,WAAW;KACZ,CAAC,CAAC;IACH,MAAM,kBAAkB,GAAoB,EAAE,CAAC;IAE/C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,CAAC,KAAK,CAAC,qCAAqC,SAAS,EAAE,CAAC,CAAC;QAC/D,MAAM,eAAe,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAErE,MAAM,kBAAkB,GAAG,WAAW,CAAC;YACrC,MAAM;YACN,OAAO;YACP,MAAM;YACN,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;YACtE,SAAS;YACT,MAAM,EAAE,OAAO,CAAC,MAAM;SACvB,CAAC;aACC,IAAI,CAAC,GAAG,EAAE;YACT,sBAAsB,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAClD,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YACtB,MAAM,CAAC,KAAK,CACV,EAAE,GAAG,EAAE,EACP,wCAAwC,SAAS,aAAa,CAC/D,CAAC;YACF,sBAAsB,CAAC,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;QACL,kBAAkB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,CAAC,IAAI,CACT,OAAO,UAAU,CAAC,MAAM,8EAA8E,CACvG,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IACtC,MAAM,sBAAsB,CAAC,OAAO,EAAE,CAAC;IACvC,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,mBAAmB,EAAE,CAAC;IAEtE,MAAM,CAAC,IAAI,CACT,EAAE,gBAAgB,EAAE,EACpB,0BAA0B,UAAU,CAAC,MAAM,mBAAmB,gBAAgB,CAAC,MAAM,YAAY,CAClG,CAAC;IAEF,MAAM,iBAAiB,GAAG,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC;IACtD,OAAO,iBAAiB,CAAC;AAC3B,CAAC"}
@@ -8,7 +8,10 @@ interface CreatorManifestManagerDeps {
8
8
  export declare class CreatorManifestManager {
9
9
  #private;
10
10
  constructor({ logger, pathManager }: CreatorManifestManagerDeps);
11
+ getFailedCreatorIds(): string[];
11
12
  load(creatorId: string): Promise<CreatorManifest>;
13
+ markFailed(creatorId: string, error: unknown): void;
14
+ markSucceeded(creatorId: string): void;
12
15
  saveAll(): Promise<void>;
13
16
  }
14
17
  export {};
@@ -1 +1 @@
1
- {"version":3,"file":"creator-manager.d.ts","sourceRoot":"","sources":["../../../../src/downloader/manifest/creator-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C,UAAU,0BAA0B;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,WAAW,CAAC;CAC1B;AAED,qBAAa,sBAAsB;;gBAKrB,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,0BAA0B;IAKzD,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IASjD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAkB/B"}
1
+ {"version":3,"file":"creator-manager.d.ts","sourceRoot":"","sources":["../../../../src/downloader/manifest/creator-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C,UAAU,0BAA0B;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,WAAW,CAAC;CAC1B;AAED,qBAAa,sBAAsB;;gBAKrB,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,0BAA0B;IAK/D,mBAAmB,IAAI,MAAM,EAAE;IASzB,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IASvD,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI;IAInD,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAIhC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CA2B/B"}