@moontaiworks/fanbox-dl 1.5.5 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/README.md +51 -37
  2. package/dist/cjs/downloader/asset/download.d.ts +3 -1
  3. package/dist/cjs/downloader/asset/download.d.ts.map +1 -1
  4. package/dist/cjs/downloader/asset/download.js +11 -14
  5. package/dist/cjs/downloader/asset/download.js.map +1 -1
  6. package/dist/cjs/downloader/cli/options.d.ts +4 -1
  7. package/dist/cjs/downloader/cli/options.d.ts.map +1 -1
  8. package/dist/cjs/downloader/cli/options.js +10 -1
  9. package/dist/cjs/downloader/cli/options.js.map +1 -1
  10. package/dist/cjs/downloader/creator/discover-posts.d.ts.map +1 -1
  11. package/dist/cjs/downloader/creator/discover-posts.js +5 -3
  12. package/dist/cjs/downloader/creator/discover-posts.js.map +1 -1
  13. package/dist/cjs/downloader/creator/sync.d.ts +2 -1
  14. package/dist/cjs/downloader/creator/sync.d.ts.map +1 -1
  15. package/dist/cjs/downloader/creator/sync.js +26 -27
  16. package/dist/cjs/downloader/creator/sync.js.map +1 -1
  17. package/dist/cjs/downloader/fs/filesystem.d.ts +3 -0
  18. package/dist/cjs/downloader/fs/filesystem.d.ts.map +1 -1
  19. package/dist/cjs/downloader/fs/filesystem.js +27 -2
  20. package/dist/cjs/downloader/fs/filesystem.js.map +1 -1
  21. package/dist/cjs/downloader/fs/path-manager.d.ts +10 -3
  22. package/dist/cjs/downloader/fs/path-manager.d.ts.map +1 -1
  23. package/dist/cjs/downloader/fs/path-manager.js +102 -12
  24. package/dist/cjs/downloader/fs/path-manager.js.map +1 -1
  25. package/dist/cjs/downloader/index.d.ts.map +1 -1
  26. package/dist/cjs/downloader/index.js +16 -13
  27. package/dist/cjs/downloader/index.js.map +1 -1
  28. package/dist/cjs/downloader/manifest/creator-manager.d.ts +3 -0
  29. package/dist/cjs/downloader/manifest/creator-manager.d.ts.map +1 -1
  30. package/dist/cjs/downloader/manifest/creator-manager.js +30 -2
  31. package/dist/cjs/downloader/manifest/creator-manager.js.map +1 -1
  32. package/dist/cjs/downloader/manifest/creator.d.ts +6 -0
  33. package/dist/cjs/downloader/manifest/creator.d.ts.map +1 -1
  34. package/dist/cjs/downloader/manifest/creator.js +11 -1
  35. package/dist/cjs/downloader/manifest/creator.js.map +1 -1
  36. package/dist/cjs/downloader/post/article.d.ts.map +1 -1
  37. package/dist/cjs/downloader/post/article.js +25 -22
  38. package/dist/cjs/downloader/post/article.js.map +1 -1
  39. package/dist/cjs/downloader/post/sync.d.ts +3 -2
  40. package/dist/cjs/downloader/post/sync.d.ts.map +1 -1
  41. package/dist/cjs/downloader/post/sync.js +127 -79
  42. package/dist/cjs/downloader/post/sync.js.map +1 -1
  43. package/dist/cjs/downloader/post/verify.d.ts +9 -0
  44. package/dist/cjs/downloader/post/verify.d.ts.map +1 -0
  45. package/dist/cjs/downloader/post/verify.js +64 -0
  46. package/dist/cjs/downloader/post/verify.js.map +1 -0
  47. package/dist/esm/downloader/asset/download.d.ts +3 -1
  48. package/dist/esm/downloader/asset/download.d.ts.map +1 -1
  49. package/dist/esm/downloader/asset/download.js +12 -15
  50. package/dist/esm/downloader/asset/download.js.map +1 -1
  51. package/dist/esm/downloader/cli/options.d.ts +4 -1
  52. package/dist/esm/downloader/cli/options.d.ts.map +1 -1
  53. package/dist/esm/downloader/cli/options.js +10 -1
  54. package/dist/esm/downloader/cli/options.js.map +1 -1
  55. package/dist/esm/downloader/creator/discover-posts.d.ts.map +1 -1
  56. package/dist/esm/downloader/creator/discover-posts.js +5 -3
  57. package/dist/esm/downloader/creator/discover-posts.js.map +1 -1
  58. package/dist/esm/downloader/creator/sync.d.ts +2 -1
  59. package/dist/esm/downloader/creator/sync.d.ts.map +1 -1
  60. package/dist/esm/downloader/creator/sync.js +26 -27
  61. package/dist/esm/downloader/creator/sync.js.map +1 -1
  62. package/dist/esm/downloader/fs/filesystem.d.ts +3 -0
  63. package/dist/esm/downloader/fs/filesystem.d.ts.map +1 -1
  64. package/dist/esm/downloader/fs/filesystem.js +24 -2
  65. package/dist/esm/downloader/fs/filesystem.js.map +1 -1
  66. package/dist/esm/downloader/fs/path-manager.d.ts +10 -3
  67. package/dist/esm/downloader/fs/path-manager.d.ts.map +1 -1
  68. package/dist/esm/downloader/fs/path-manager.js +102 -12
  69. package/dist/esm/downloader/fs/path-manager.js.map +1 -1
  70. package/dist/esm/downloader/index.d.ts.map +1 -1
  71. package/dist/esm/downloader/index.js +16 -13
  72. package/dist/esm/downloader/index.js.map +1 -1
  73. package/dist/esm/downloader/manifest/creator-manager.d.ts +3 -0
  74. package/dist/esm/downloader/manifest/creator-manager.d.ts.map +1 -1
  75. package/dist/esm/downloader/manifest/creator-manager.js +30 -2
  76. package/dist/esm/downloader/manifest/creator-manager.js.map +1 -1
  77. package/dist/esm/downloader/manifest/creator.d.ts +6 -0
  78. package/dist/esm/downloader/manifest/creator.d.ts.map +1 -1
  79. package/dist/esm/downloader/manifest/creator.js +11 -1
  80. package/dist/esm/downloader/manifest/creator.js.map +1 -1
  81. package/dist/esm/downloader/post/article.d.ts.map +1 -1
  82. package/dist/esm/downloader/post/article.js +25 -22
  83. package/dist/esm/downloader/post/article.js.map +1 -1
  84. package/dist/esm/downloader/post/sync.d.ts +3 -2
  85. package/dist/esm/downloader/post/sync.d.ts.map +1 -1
  86. package/dist/esm/downloader/post/sync.js +127 -79
  87. package/dist/esm/downloader/post/sync.js.map +1 -1
  88. package/dist/esm/downloader/post/verify.d.ts +9 -0
  89. package/dist/esm/downloader/post/verify.d.ts.map +1 -0
  90. package/dist/esm/downloader/post/verify.js +61 -0
  91. package/dist/esm/downloader/post/verify.js.map +1 -0
  92. package/package.json +1 -1
@@ -8,7 +8,17 @@ const download_js_1 = require("../asset/download.js");
8
8
  const asset_js_1 = require("../markdown/asset.js");
9
9
  const content_js_1 = require("./content.js");
10
10
  const contents_js_1 = require("./contents.js");
11
- function preSyncPostCheck({ logger, manifest }, postSummary) {
11
+ const verify_js_1 = require("./verify.js");
12
+ async function preSyncPostCheck({ logger, manifest, verify }, postSummary) {
13
+ const existingPost = manifest.posts[postSummary.id];
14
+ if (existingPost?.status === "complete" &&
15
+ postSummary.updatedDatetime === existingPost.updatedDatetime) {
16
+ logger.debug(`Post ${postSummary.id} has no changes, skipping download.`);
17
+ if (verify) {
18
+ return (0, verify_js_1.verifyCompletePost)({ logger }, existingPost, postSummary);
19
+ }
20
+ return existingPost;
21
+ }
12
22
  if (postSummary.isRestricted) {
13
23
  logger.debug(`Post ${postSummary.id} is restricted, skipping download.`);
14
24
  return {
@@ -19,12 +29,6 @@ function preSyncPostCheck({ logger, manifest }, postSummary) {
19
29
  updatedDatetime: postSummary.updatedDatetime,
20
30
  };
21
31
  }
22
- const existingPost = manifest.posts[postSummary.id];
23
- if (existingPost?.status === "complete" &&
24
- postSummary.updatedDatetime === existingPost.updatedDatetime) {
25
- logger.debug(`Post ${postSummary.id} has no changes, skipping download.`);
26
- return existingPost;
27
- }
28
32
  return {
29
33
  assets: {},
30
34
  id: postSummary.id,
@@ -38,86 +42,31 @@ async function syncPost({ headers, logger, pathManager, transport }, post) {
38
42
  const contents = (0, contents_js_1.formatPostContents)({ logger }, post);
39
43
  if (post.coverImageUrl)
40
44
  contents.unshift(formatCoverImage(post));
41
- let hasUnknownContent = false;
42
- const assetManifestData = {};
43
- const download = download_js_1.downloadAsset.bind(undefined, {
44
- headers,
45
- logger,
46
- transport,
47
- });
48
45
  const totalDigits = contents.length.toString().length;
49
- const results = await Promise.allSettled(contents.map(async (content, index) => {
50
- const indexPadded = index.toString().padStart(totalDigits, "0");
51
- if (content instanceof content_js_1.ImageContent) {
52
- const destination = pathManager.asset(indexPadded, content.id, content.extension);
53
- const { bytes, sha256 } = await download({
54
- destination,
55
- fallbackDateTime: post.updatedDatetime,
56
- mediaContent: content,
57
- }).catch((err) => {
58
- logger.error({ err }, `Error occurred while downloading image asset ${content.id}, skipping.`);
59
- assetManifestData[content.id] = {
60
- path: destination,
61
- status: "failed",
62
- url: content.url,
63
- };
64
- throw err;
65
- });
66
- assetManifestData[content.id] = {
67
- bytes,
68
- path: destination,
69
- sha256,
70
- status: "complete",
71
- url: content.url,
72
- };
73
- return (0, asset_js_1.formatImageAsset)({ assetPath: destination });
74
- }
75
- if (content instanceof content_js_1.FileContent) {
76
- const destination = pathManager.asset(indexPadded, `${content.name}-${content.id}`, content.extension);
77
- const { bytes, sha256 } = await download({
78
- destination,
79
- fallbackDateTime: post.updatedDatetime,
80
- mediaContent: content,
81
- }).catch((err) => {
82
- logger.error({ err }, `Error occurred while downloading file asset ${content.id}, skipping.`);
83
- assetManifestData[content.id] = {
84
- path: destination,
85
- status: "failed",
86
- url: content.url,
87
- };
88
- throw err;
89
- });
90
- assetManifestData[content.id] = {
91
- bytes,
92
- path: destination,
93
- sha256,
94
- status: "complete",
95
- url: content.url,
96
- };
97
- return (0, asset_js_1.formatFileAsset)({ assetPath: destination });
98
- }
99
- if (content instanceof content_js_1.TextContent) {
100
- return (0, asset_js_1.formatTextContent)(content);
101
- }
102
- logger.warn(`Post ${post.id} has unhandled content type ${content.type}, skipping.`);
103
- hasUnknownContent = true;
104
- }));
105
- const markdownContent = results.map((result) => {
106
- if (result.status === "fulfilled") {
107
- return result.value;
108
- }
109
- const err = String(result.reason);
110
- return `<!-- Error occurred while processing content: ${err} -->`;
111
- });
112
- await (0, promises_1.writeFile)(pathManager.asset("0".padStart(totalDigits, "0"), "content", "md"), markdownContent.join("\n\n"));
46
+ const results = await Promise.allSettled(contents.map((content, index) => processContent({ headers, logger, pathManager, post, transport }, { content, index, totalDigits })));
47
+ const hasRejectedContent = results.some((result) => result.status === "rejected");
48
+ const fulfilled = fulfilledResults(results);
49
+ const hasUnknownContent = fulfilled.some((result) => result.unknown);
50
+ const hasFailedContent = fulfilled.some((result) => result.failed);
51
+ const assetManifestData = collectAssetManifestData(fulfilled);
52
+ const markdownContent = formatMarkdownContent(results);
53
+ await (0, promises_1.writeFile)(pathManager.asset([
54
+ { context: "0".padStart(totalDigits, "0"), required: true },
55
+ { context: "content", required: true },
56
+ ], "md"), markdownContent.join("\n\n"));
113
57
  return {
114
58
  assets: assetManifestData,
115
59
  id: post.id,
116
60
  restricted: false,
117
- status: hasUnknownContent ? "partial" : "complete",
61
+ status: hasUnknownContent || hasFailedContent || hasRejectedContent
62
+ ? "partial"
63
+ : "complete",
118
64
  updatedDatetime: post.updatedDatetime,
119
65
  };
120
66
  }
67
+ function collectAssetManifestData(contents) {
68
+ return Object.fromEntries(contents.flatMap((content) => content.assets ?? []));
69
+ }
121
70
  function formatCoverImage(post) {
122
71
  if (!post.coverImageUrl)
123
72
  throw new Error("Post does not have a cover image");
@@ -128,4 +77,103 @@ function formatCoverImage(post) {
128
77
  originalUrl: post.coverImageUrl,
129
78
  });
130
79
  }
80
+ function formatMarkdownContent(results) {
81
+ return results.map((result) => {
82
+ if (result.status === "fulfilled") {
83
+ return result.value.markdown;
84
+ }
85
+ const err = String(result.reason);
86
+ return `<!-- Error occurred while processing content: ${err} -->`;
87
+ });
88
+ }
89
+ function formatMediaAsset(content, destination) {
90
+ if (content instanceof content_js_1.ImageContent) {
91
+ return (0, asset_js_1.formatImageAsset)({ assetPath: destination });
92
+ }
93
+ return (0, asset_js_1.formatFileAsset)({ assetPath: destination });
94
+ }
95
+ function fulfilledResults(results) {
96
+ return results
97
+ .filter((result) => result.status === "fulfilled")
98
+ .map((result) => result.value);
99
+ }
100
+ function isMediaContent(content) {
101
+ return content instanceof content_js_1.FileContent || content instanceof content_js_1.ImageContent;
102
+ }
103
+ function mediaAssetSegments(content, index, totalDigits) {
104
+ const indexPadded = index.toString().padStart(totalDigits, "0");
105
+ if (content instanceof content_js_1.FileContent) {
106
+ return [
107
+ { context: indexPadded, required: true },
108
+ { context: content.name, required: false },
109
+ { context: content.id, required: true },
110
+ ];
111
+ }
112
+ return [
113
+ { context: indexPadded, required: true },
114
+ { context: content.id, required: true },
115
+ ];
116
+ }
117
+ async function processContent({ headers, logger, pathManager, post, transport }, { content, index, totalDigits }) {
118
+ if (isMediaContent(content)) {
119
+ return syncMediaContent({ headers, logger, pathManager, transport }, {
120
+ content,
121
+ fallbackDateTime: post.updatedDatetime,
122
+ index,
123
+ totalDigits,
124
+ });
125
+ }
126
+ if (content instanceof content_js_1.TextContent) {
127
+ return { markdown: (0, asset_js_1.formatTextContent)(content) };
128
+ }
129
+ logger.warn(`Post ${post.id} has unhandled content type ${content.type}, skipping.`);
130
+ return { unknown: true };
131
+ }
132
+ async function syncMediaContent({ headers, logger, pathManager, transport, }, { content, fallbackDateTime, index, totalDigits, }) {
133
+ const destination = pathManager.asset(mediaAssetSegments(content, index, totalDigits), content.extension);
134
+ const downloadResult = await (0, download_js_1.downloadAsset)({ headers, logger, transport }, {
135
+ destination,
136
+ fallbackDateTime,
137
+ mediaContent: content,
138
+ timeOffset: index,
139
+ })
140
+ .then(({ bytes, modifiedTime, sha256 }) => ({ bytes, failed: false, modifiedTime, sha256 }))
141
+ .catch((err) => {
142
+ logger.error({ err }, `Error occurred while downloading ${content.type} asset ${content.id}, skipping.`);
143
+ return { error: err, failed: true };
144
+ });
145
+ if (downloadResult.failed) {
146
+ return {
147
+ assets: [
148
+ [
149
+ content.id,
150
+ {
151
+ path: destination,
152
+ status: "failed",
153
+ url: content.url,
154
+ },
155
+ ],
156
+ ],
157
+ failed: true,
158
+ markdown: `<!-- Error occurred while processing content: ${String(downloadResult.error)} -->`,
159
+ };
160
+ }
161
+ return {
162
+ assets: [
163
+ [
164
+ content.id,
165
+ {
166
+ bytes: downloadResult.bytes,
167
+ contentIndex: index,
168
+ expectedTime: downloadResult.modifiedTime,
169
+ path: destination,
170
+ sha256: downloadResult.sha256,
171
+ status: "complete",
172
+ url: content.url,
173
+ },
174
+ ],
175
+ ],
176
+ markdown: formatMediaAsset(content, destination),
177
+ };
178
+ }
131
179
  //# sourceMappingURL=sync.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"sync.js","sourceRoot":"","sources":["../../../../src/downloader/post/sync.ts"],"names":[],"mappings":";;AAkCA,4CAgCC;AAED,4BAmIC;AAvMD,+CAA6C;AAC7C,yCAAoC;AAMpC,sDAAqD;AAOrD,mDAI8B;AAC9B,6CAAsE;AACtE,+CAAmD;AAcnD,SAAgB,gBAAgB,CAC9B,EAAE,MAAM,EAAE,QAAQ,EAAwB,EAC1C,WAAwB;IAExB,IAAI,WAAW,CAAC,YAAY,EAAE,CAAC;QAE7B,MAAM,CAAC,KAAK,CAAC,QAAQ,WAAW,CAAC,EAAE,oCAAoC,CAAC,CAAC;QACzE,OAAO;YACL,MAAM,EAAE,EAAE;YACV,EAAE,EAAE,WAAW,CAAC,EAAE;YAClB,UAAU,EAAE,IAAI;YAChB,MAAM,EAAE,SAAS;YACjB,eAAe,EAAE,WAAW,CAAC,eAAe;SAC7C,CAAC;IACJ,CAAC;IACD,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IACpD,IACE,YAAY,EAAE,MAAM,KAAK,UAAU;QACnC,WAAW,CAAC,eAAe,KAAK,YAAY,CAAC,eAAe,EAC5D,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,QAAQ,WAAW,CAAC,EAAE,qCAAqC,CAAC,CAAC;QAC1E,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,OAAO;QACL,MAAM,EAAE,EAAE;QACV,EAAE,EAAE,WAAW,CAAC,EAAE;QAClB,UAAU,EAAE,KAAK;QACjB,MAAM,EAAE,SAAS;QACjB,eAAe,EAAE,WAAW,CAAC,eAAe;KAC7C,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,QAAQ,CAC5B,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAgB,EACzD,IAAU;IAEV,MAAM,CAAC,IAAI,CACT,oBAAoB,IAAI,CAAC,EAAE,eAAe,IAAI,CAAC,eAAe,eAAe,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,KAAK,EAAE,CAC7G,CAAC;IACF,MAAM,QAAQ,GAAG,IAAA,gCAAkB,EAAC,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,CAAC;IACtD,IAAI,IAAI,CAAC,aAAa;QAAE,QAAQ,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;IAEjE,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAC9B,MAAM,iBAAiB,GAAsC,EAAE,CAAC;IAChE,MAAM,QAAQ,GAAG,2BAAa,CAAC,IAAI,CAAC,SAAS,EAAE;QAC7C,OAAO;QACP,MAAM;QACN,SAAS;KACV,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC;IACtD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACpC,MAAM,WAAW,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QAEhE,IAAI,OAAO,YAAY,yBAAY,EAAE,CAAC;YACpC,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CACnC,WAAW,EACX,OAAO,CAAC,EAAE,EACV,OAAO,CAAC,SAAS,CAClB,CAAC;YAEF,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;gBACvC,WAAW;gBACX,gBAAgB,EAAE,IAAI,CAAC,eAAe;gBACtC,YAAY,EAAE,OAAO;aACtB,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;gBACxB,MAAM,CAAC,KAAK,CACV,EAAE,GAAG,EAAE,EACP,gDAAgD,OAAO,CAAC,EAAE,aAAa,CACxE,CAAC;gBAEF,iBAAiB,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG;oBAC9B,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,QAAQ;oBAChB,GAAG,EAAE,OAAO,CAAC,GAAG;iBACjB,CAAC;gBAEF,MAAM,GAAG,CAAC;YACZ,CAAC,CAAC,CAAC;YAEH,iBAAiB,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG;gBAC9B,KAAK;gBACL,IAAI,EAAE,WAAW;gBACjB,MAAM;gBACN,MAAM,EAAE,UAAU;gBAClB,GAAG,EAAE,OAAO,CAAC,GAAG;aACjB,CAAC;YAEF,OAAO,IAAA,2BAAgB,EAAC,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,CAAC;QACtD,CAAC;QAED,IAAI,OAAO,YAAY,wBAAW,EAAE,CAAC;YACnC,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CACnC,WAAW,EACX,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,EAAE,EAC/B,OAAO,CAAC,SAAS,CAClB,CAAC;YAEF,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;gBACvC,WAAW;gBACX,gBAAgB,EAAE,IAAI,CAAC,eAAe;gBACtC,YAAY,EAAE,OAAO;aACtB,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;gBACxB,MAAM,CAAC,KAAK,CACV,EAAE,GAAG,EAAE,EACP,+CAA+C,OAAO,CAAC,EAAE,aAAa,CACvE,CAAC;gBAEF,iBAAiB,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG;oBAC9B,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,QAAQ;oBAChB,GAAG,EAAE,OAAO,CAAC,GAAG;iBACjB,CAAC;gBAEF,MAAM,GAAG,CAAC;YACZ,CAAC,CAAC,CAAC;YAEH,iBAAiB,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG;gBAC9B,KAAK;gBACL,IAAI,EAAE,WAAW;gBACjB,MAAM;gBACN,MAAM,EAAE,UAAU;gBAClB,GAAG,EAAE,OAAO,CAAC,GAAG;aACjB,CAAC;YAEF,OAAO,IAAA,0BAAe,EAAC,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,IAAI,OAAO,YAAY,wBAAW,EAAE,CAAC;YACnC,OAAO,IAAA,4BAAiB,EAAC,OAAO,CAAC,CAAC;QACpC,CAAC;QAED,MAAM,CAAC,IAAI,CACT,QAAQ,IAAI,CAAC,EAAE,+BAA+B,OAAO,CAAC,IAAI,aAAa,CACxE,CAAC;QACF,iBAAiB,GAAG,IAAI,CAAC;IAC3B,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAClC,OAAO,MAAM,CAAC,KAAK,CAAC;QACtB,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClC,OAAO,iDAAiD,GAAG,MAAM,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,MAAM,IAAA,oBAAS,EACb,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,EAClE,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAC7B,CAAC;IAEF,OAAO;QACL,MAAM,EAAE,iBAAiB;QACzB,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,UAAU,EAAE,KAAK;QAGjB,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU;QAClD,eAAe,EAAE,IAAI,CAAC,eAAe;KACtC,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAU;IAClC,IAAI,CAAC,IAAI,CAAC,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAE7E,MAAM,SAAS,GACb,IAAA,mBAAO,EAAC,IAAI,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;IAElE,OAAO,IAAI,yBAAY,CAAC;QACtB,SAAS;QACT,EAAE,EAAE,OAAO;QACX,WAAW,EAAE,IAAI,CAAC,aAAa;KAChC,CAAC,CAAC;AACL,CAAC"}
1
+ {"version":3,"file":"sync.js","sourceRoot":"","sources":["../../../../src/downloader/post/sync.ts"],"names":[],"mappings":";;AAiEA,4CAqCC;AAED,4BAiDC;AAzJD,+CAA6C;AAC7C,yCAAoC;AAMpC,sDAAqD;AAOrD,mDAI8B;AAC9B,6CAAsE;AAEtE,+CAAmD;AACnD,2CAAiD;AA2C1C,KAAK,UAAU,gBAAgB,CACpC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAwB,EAClD,WAAwB;IAExB,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IACpD,IACE,YAAY,EAAE,MAAM,KAAK,UAAU;QACnC,WAAW,CAAC,eAAe,KAAK,YAAY,CAAC,eAAe,EAC5D,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,QAAQ,WAAW,CAAC,EAAE,qCAAqC,CAAC,CAAC;QAC1E,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,IAAA,8BAAkB,EAAC,EAAE,MAAM,EAAE,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;QACnE,CAAC;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,IAAI,WAAW,CAAC,YAAY,EAAE,CAAC;QAE7B,MAAM,CAAC,KAAK,CAAC,QAAQ,WAAW,CAAC,EAAE,oCAAoC,CAAC,CAAC;QACzE,OAAO;YACL,MAAM,EAAE,EAAE;YACV,EAAE,EAAE,WAAW,CAAC,EAAE;YAClB,UAAU,EAAE,IAAI;YAChB,MAAM,EAAE,SAAS;YACjB,eAAe,EAAE,WAAW,CAAC,eAAe;SAC7C,CAAC;IACJ,CAAC;IAED,OAAO;QACL,MAAM,EAAE,EAAE;QACV,EAAE,EAAE,WAAW,CAAC,EAAE;QAClB,UAAU,EAAE,KAAK;QACjB,MAAM,EAAE,SAAS;QACjB,eAAe,EAAE,WAAW,CAAC,eAAe;KAC7C,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,QAAQ,CAC5B,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAgB,EACzD,IAAU;IAEV,MAAM,CAAC,IAAI,CACT,oBAAoB,IAAI,CAAC,EAAE,eAAe,IAAI,CAAC,eAAe,eAAe,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,KAAK,EAAE,CAC7G,CAAC;IACF,MAAM,QAAQ,GAAG,IAAA,gCAAkB,EAAC,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,CAAC;IACtD,IAAI,IAAI,CAAC,aAAa;QAAE,QAAQ,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;IAEjE,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC;IACtD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAC9B,cAAc,CACZ,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,EACjD,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,CAChC,CACF,CACF,CAAC;IACF,MAAM,kBAAkB,GAAG,OAAO,CAAC,IAAI,CACrC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,UAAU,CACzC,CAAC;IACF,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,iBAAiB,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACrE,MAAM,gBAAgB,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACnE,MAAM,iBAAiB,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC;IAC9D,MAAM,eAAe,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;IAEvD,MAAM,IAAA,oBAAS,EACb,WAAW,CAAC,KAAK,CACf;QACE,EAAE,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE;QAC3D,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE;KACvC,EACD,IAAI,CACL,EACD,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAC7B,CAAC;IAEF,OAAO;QACL,MAAM,EAAE,iBAAiB;QACzB,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,UAAU,EAAE,KAAK;QACjB,MAAM,EACJ,iBAAiB,IAAI,gBAAgB,IAAI,kBAAkB;YACzD,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,UAAU;QAChB,eAAe,EAAE,IAAI,CAAC,eAAe;KACtC,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAC/B,QAA4B;IAE5B,OAAO,MAAM,CAAC,WAAW,CACvB,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CACpD,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAU;IAClC,IAAI,CAAC,IAAI,CAAC,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAE7E,MAAM,SAAS,GACb,IAAA,mBAAO,EAAC,IAAI,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;IAElE,OAAO,IAAI,yBAAY,CAAC;QACtB,SAAS;QACT,EAAE,EAAE,OAAO;QACX,WAAW,EAAE,IAAI,CAAC,aAAa;KAChC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,qBAAqB,CAC5B,OAAiD;IAEjD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC5B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAClC,OAAO,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC/B,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClC,OAAO,iDAAiD,GAAG,MAAM,CAAC;IACpE,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAqB,EAAE,WAAmB;IAClE,IAAI,OAAO,YAAY,yBAAY,EAAE,CAAC;QACpC,OAAO,IAAA,2BAAgB,EAAC,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,IAAA,0BAAe,EAAC,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,gBAAgB,CACvB,OAAiD;IAEjD,OAAO,OAAO;SACX,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC;SACjD,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,cAAc,CACrB,OAAgB;IAEhB,OAAO,OAAO,YAAY,wBAAW,IAAI,OAAO,YAAY,yBAAY,CAAC;AAC3E,CAAC;AAED,SAAS,kBAAkB,CACzB,OAAqB,EACrB,KAAa,EACb,WAAmB;IAEnB,MAAM,WAAW,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;IAEhE,IAAI,OAAO,YAAY,wBAAW,EAAE,CAAC;QACnC,OAAO;YACL,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE;YACxC,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE;YAC1C,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;SACxC,CAAC;IACJ,CAAC;IAED,OAAO;QACL,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE;QACxC,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;KACxC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAsB,EACrE,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAyB;IAEtD,IAAI,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,OAAO,gBAAgB,CACrB,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,EAC3C;YACE,OAAO;YACP,gBAAgB,EAAE,IAAI,CAAC,eAAe;YACtC,KAAK;YACL,WAAW;SACZ,CACF,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,YAAY,wBAAW,EAAE,CAAC;QACnC,OAAO,EAAE,QAAQ,EAAE,IAAA,4BAAiB,EAAC,OAAO,CAAC,EAAE,CAAC;IAClD,CAAC;IAED,MAAM,CAAC,IAAI,CACT,QAAQ,IAAI,CAAC,EAAE,+BAA+B,OAAO,CAAC,IAAI,aAAa,CACxE,CAAC;IACF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC3B,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,EACE,OAAO,EACP,MAAM,EACN,WAAW,EACX,SAAS,GAMV,EACD,EACE,OAAO,EACP,gBAAgB,EAChB,KAAK,EACL,WAAW,GAMZ;IAED,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CACnC,kBAAkB,CAAC,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,EAC/C,OAAO,CAAC,SAAS,CAClB,CAAC;IACF,MAAM,cAAc,GAAmB,MAAM,IAAA,2BAAa,EACxD,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,EAC9B;QACE,WAAW;QACX,gBAAgB;QAChB,YAAY,EAAE,OAAO;QACrB,UAAU,EAAE,KAAK;KAClB,CACF;SACE,IAAI,CACH,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE,CAClC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,CAAU,CAC5D;SACA,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;QACtB,MAAM,CAAC,KAAK,CACV,EAAE,GAAG,EAAE,EACP,oCAAoC,OAAO,CAAC,IAAI,UAAU,OAAO,CAAC,EAAE,aAAa,CAClF,CAAC;QAEF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAW,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEL,IAAI,cAAc,CAAC,MAAM,EAAE,CAAC;QAC1B,OAAO;YACL,MAAM,EAAE;gBACN;oBACE,OAAO,CAAC,EAAE;oBACV;wBACE,IAAI,EAAE,WAAW;wBACjB,MAAM,EAAE,QAAQ;wBAChB,GAAG,EAAE,OAAO,CAAC,GAAG;qBACjB;iBACF;aACF;YACD,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,iDAAiD,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM;SACnE,CAAC;IAC/B,CAAC;IAED,OAAO;QACL,MAAM,EAAE;YACN;gBACE,OAAO,CAAC,EAAE;gBACV;oBACE,KAAK,EAAE,cAAc,CAAC,KAAK;oBAC3B,YAAY,EAAE,KAAK;oBACnB,YAAY,EAAE,cAAc,CAAC,YAAY;oBACzC,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,cAAc,CAAC,MAAM;oBAC7B,MAAM,EAAE,UAAU;oBAClB,GAAG,EAAE,OAAO,CAAC,GAAG;iBACjB;aACF;SACF;QACD,QAAQ,EAAE,gBAAgB,CAAC,OAAO,EAAE,WAAW,CAAC;KACtB,CAAC;AAC/B,CAAC"}
@@ -0,0 +1,9 @@
1
+ import type { Logger } from "pino";
2
+ import type { PostSummary } from "../../client/types.js";
3
+ import type { PostManifestData } from "../manifest/creator.js";
4
+ interface VerifyPostCheckDeps {
5
+ logger: Logger;
6
+ }
7
+ export declare function verifyCompletePost({ logger }: VerifyPostCheckDeps, manifest: PostManifestData, postSummary: PostSummary): Promise<PostManifestData>;
8
+ export {};
9
+ //# sourceMappingURL=verify.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"verify.d.ts","sourceRoot":"","sources":["../../../../src/downloader/post/verify.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAOzD,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,wBAAwB,CAAC;AAEhC,UAAU,mBAAmB;IAC3B,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,kBAAkB,CACtC,EAAE,MAAM,EAAE,EAAE,mBAAmB,EAC/B,QAAQ,EAAE,gBAAgB,EAC1B,WAAW,EAAE,WAAW,GACvB,OAAO,CAAC,gBAAgB,CAAC,CAqB3B"}
@@ -0,0 +1,64 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.verifyCompletePost = verifyCompletePost;
4
+ const promises_1 = require("node:fs/promises");
5
+ const filesystem_js_1 = require("../fs/filesystem.js");
6
+ async function verifyCompletePost({ logger }, manifest, postSummary) {
7
+ const assets = Object.fromEntries(await Promise.all(Object.entries(manifest.assets).map(async ([id, asset]) => [
8
+ id,
9
+ asset ? await verifyAsset({ logger }, asset, postSummary) : asset,
10
+ ])));
11
+ const hasFailedAsset = Object.values(assets).some((asset) => asset && asset.status !== "complete");
12
+ return {
13
+ ...manifest,
14
+ assets,
15
+ status: hasFailedAsset ? "partial" : "complete",
16
+ };
17
+ }
18
+ function expectedModifiedTime(asset, postSummary) {
19
+ if (asset.expectedTime) {
20
+ const modifiedTime = (0, filesystem_js_1.normalizeFileTimestamp)(new Date(asset.expectedTime));
21
+ if (!Number.isNaN(modifiedTime.getTime()))
22
+ return modifiedTime;
23
+ }
24
+ return (0, filesystem_js_1.normalizeFileTimestamp)(new Date(new Date(postSummary.updatedDatetime).getTime() +
25
+ asset.contentIndex * 1000));
26
+ }
27
+ function failedAsset(asset, error) {
28
+ return {
29
+ ...asset,
30
+ error,
31
+ status: "failed",
32
+ };
33
+ }
34
+ function obsoleteAsset(asset, error) {
35
+ return {
36
+ ...asset,
37
+ error,
38
+ status: "obsolete",
39
+ };
40
+ }
41
+ async function verifyAsset({ logger }, asset, postSummary) {
42
+ if (asset.status !== "complete") {
43
+ return asset;
44
+ }
45
+ if (asset.contentIndex === undefined) {
46
+ return obsoleteAsset(asset, "content index missing");
47
+ }
48
+ if (!(await (0, filesystem_js_1.exists)(asset.path))) {
49
+ return failedAsset(asset, "file missing");
50
+ }
51
+ const actualSha256 = await (0, filesystem_js_1.hashFile)(asset.path);
52
+ if (actualSha256 !== asset.sha256) {
53
+ return failedAsset(asset, "sha256 mismatch");
54
+ }
55
+ const expectedTime = expectedModifiedTime(asset, postSummary);
56
+ const { mtimeMs } = await (0, promises_1.stat)(asset.path);
57
+ if ((0, filesystem_js_1.normalizeFileTimestamp)(new Date(mtimeMs)).getTime() !==
58
+ expectedTime.getTime()) {
59
+ logger.debug(`Repairing modified time of verified asset ${asset.path} from ${new Date(mtimeMs).toISOString()} to ${expectedTime.toISOString()}`);
60
+ await (0, promises_1.utimes)(asset.path, expectedTime, expectedTime);
61
+ }
62
+ return { ...asset, expectedTime: (0, filesystem_js_1.formatFileTimestamp)(expectedTime) };
63
+ }
64
+ //# sourceMappingURL=verify.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"verify.js","sourceRoot":"","sources":["../../../../src/downloader/post/verify.ts"],"names":[],"mappings":";;AAoBA,gDAyBC;AA7CD,+CAAgD;AAKhD,uDAK6B;AAUtB,KAAK,UAAU,kBAAkB,CACtC,EAAE,MAAM,EAAuB,EAC/B,QAA0B,EAC1B,WAAwB;IAExB,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAC/B,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,CACjC,KAAK,EAAE,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CACpB;QACE,EAAE;QACF,KAAK,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK;KAChB,CACtD,CACF,CACF,CAAC;IACF,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAC/C,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU,CAChD,CAAC;IAEF,OAAO;QACL,GAAG,QAAQ;QACX,MAAM;QACN,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU;KAChD,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,KAAwB,EACxB,WAAwB;IAExB,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;QACvB,MAAM,YAAY,GAAG,IAAA,sCAAsB,EAAC,IAAI,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC;QAC1E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;YAAE,OAAO,YAAY,CAAC;IACjE,CAAC;IAED,OAAO,IAAA,sCAAsB,EAC3B,IAAI,IAAI,CACN,IAAI,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE;QAC7C,KAAK,CAAC,YAAa,GAAG,IAAK,CAC9B,CACF,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAClB,KAAwB,EACxB,KAAa;IAEb,OAAO;QACL,GAAG,KAAK;QACR,KAAK;QACL,MAAM,EAAE,QAAQ;KACjB,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,KAAwB,EACxB,KAAa;IAEb,OAAO;QACL,GAAG,KAAK;QACR,KAAK;QACL,MAAM,EAAE,UAAU;KACnB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,EAAE,MAAM,EAAuB,EAC/B,KAAwB,EACxB,WAAwB;IAExB,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACrC,OAAO,aAAa,CAAC,KAAK,EAAE,uBAAuB,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,CAAC,CAAC,MAAM,IAAA,sBAAM,EAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAChC,OAAO,WAAW,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,IAAA,wBAAQ,EAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChD,IAAI,YAAY,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QAClC,OAAO,WAAW,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAC9D,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAA,eAAI,EAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,IACE,IAAA,sCAAsB,EAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE;QACnD,YAAY,CAAC,OAAO,EAAE,EACtB,CAAC;QACD,MAAM,CAAC,KAAK,CACV,6CAA6C,KAAK,CAAC,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,OAAO,YAAY,CAAC,WAAW,EAAE,EAAE,CACnI,CAAC;QACF,MAAM,IAAA,iBAAM,EAAC,KAAK,CAAC,IAAI,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC;IACvD,CAAC;IAED,OAAO,EAAE,GAAG,KAAK,EAAE,YAAY,EAAE,IAAA,mCAAmB,EAAC,YAAY,CAAC,EAAE,CAAC;AACvE,CAAC"}
@@ -10,9 +10,11 @@ interface DownloadAssetOptions {
10
10
  destination: string;
11
11
  fallbackDateTime: string;
12
12
  mediaContent: MediaContent;
13
+ timeOffset: number;
13
14
  }
14
- export declare function downloadAsset({ headers, logger, transport }: DownloadAssetDeps, { destination, fallbackDateTime, mediaContent }: DownloadAssetOptions): Promise<{
15
+ export declare function downloadAsset({ headers, logger, transport }: DownloadAssetDeps, { destination, fallbackDateTime, mediaContent, timeOffset, }: DownloadAssetOptions): Promise<{
15
16
  bytes: number;
17
+ modifiedTime: string;
16
18
  sha256: string;
17
19
  }>;
18
20
  export {};
@@ -1 +1 @@
1
- {"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../../src/downloader/asset/download.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,UAAU,iBAAiB;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,aAAa,CAAC;CAC1B;AAED,UAAU,oBAAoB;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,YAAY,CAAC;CAC5B;AAkBD,wBAAsB,aAAa,CACjC,EAAE,OAAY,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,iBAAiB,EACtD,EAAE,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAE,EAAE,oBAAoB;;;GAoDtE"}
1
+ {"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../../src/downloader/asset/download.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAO9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,UAAU,iBAAiB;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,aAAa,CAAC;CAC1B;AAED,UAAU,oBAAoB;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,YAAY,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;CACpB;AAkBD,wBAAsB,aAAa,CACjC,EAAE,OAAY,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,iBAAiB,EACtD,EACE,WAAW,EACX,gBAAgB,EAChB,YAAY,EACZ,UAAU,GACX,EAAE,oBAAoB;;;;GAwDxB"}
@@ -1,9 +1,8 @@
1
- import { createHash } from "node:crypto";
2
- import { createReadStream, createWriteStream } from "node:fs";
3
- import { mkdir, rename, utimes } from "node:fs/promises";
1
+ import { createWriteStream } from "node:fs";
2
+ import { mkdir, rename, stat, utimes } from "node:fs/promises";
4
3
  import path from "node:path";
5
4
  import { pipeline } from "node:stream/promises";
6
- import { exists, filesize } from "../fs/filesystem.js";
5
+ import { exists, filesize, formatFileTimestamp, hashFile, } from "../fs/filesystem.js";
7
6
  class AssetDownloadError extends Error {
8
7
  constructor(response, url, body) {
9
8
  super(`Asset download failed: ${response.status} ${url}`);
@@ -14,14 +13,14 @@ class AssetDownloadError extends Error {
14
13
  this.url = url;
15
14
  }
16
15
  }
17
- export async function downloadAsset({ headers = {}, logger, transport }, { destination, fallbackDateTime, mediaContent }) {
16
+ export async function downloadAsset({ headers = {}, logger, transport }, { destination, fallbackDateTime, mediaContent, timeOffset, }) {
18
17
  if (await exists(destination)) {
19
18
  logger.debug(`Asset ${mediaContent.id} already exists at ${destination}`);
20
- const [{ size: bytes }, sha256] = await Promise.all([
21
- filesize(destination),
19
+ const [{ mtime, size: bytes }, sha256] = await Promise.all([
20
+ stat(destination),
22
21
  hashFile(destination),
23
22
  ]);
24
- return { bytes, sha256 };
23
+ return { bytes, modifiedTime: formatFileTimestamp(mtime), sha256 };
25
24
  }
26
25
  logger.debug(`Downloading ${mediaContent.type} asset ${mediaContent.id} to ${destination}`);
27
26
  await mkdir(path.dirname(destination), { recursive: true });
@@ -38,20 +37,18 @@ export async function downloadAsset({ headers = {}, logger, transport }, { desti
38
37
  }
39
38
  await write(tempFilePath, response, downloadedBytes);
40
39
  const modified = response.headers.get("Last-Modified");
41
- const timestamp = modified ? new Date(modified) : new Date(fallbackDateTime);
40
+ const timestamp = offsetTimestamp(modified ? new Date(modified) : new Date(fallbackDateTime), timeOffset);
42
41
  await utimes(tempFilePath, timestamp, timestamp);
43
42
  const [{ size: bytes }, sha256] = await Promise.all([
44
43
  filesize(tempFilePath),
45
44
  hashFile(tempFilePath),
46
45
  ]);
47
46
  await rename(tempFilePath, destination);
48
- return { bytes, sha256 };
47
+ const { mtime } = await stat(destination);
48
+ return { bytes, modifiedTime: formatFileTimestamp(mtime), sha256 };
49
49
  }
50
- async function hashFile(filePath) {
51
- const hash = createHash("sha256");
52
- for await (const chunk of createReadStream(filePath))
53
- hash.update(chunk);
54
- return hash.digest("hex");
50
+ function offsetTimestamp(timestamp, timeOffset) {
51
+ return new Date(timestamp.getTime() + timeOffset * 1000);
55
52
  }
56
53
  async function write(path, response, partialBytes) {
57
54
  return pipeline(response.body ?? "", createWriteStream(path, {
@@ -1 +1 @@
1
- {"version":3,"file":"download.js","sourceRoot":"","sources":["../../../../src/downloader/asset/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC9D,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAKhD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAevD,MAAM,kBAAmB,SAAQ,KAAK;IAMpC,YAAmB,QAAkB,EAAE,GAAW,EAAE,IAAa;QAC/D,KAAK,CAAC,0BAA0B,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;QACtC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,EAAE,OAAO,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,EAAqB,EACtD,EAAE,WAAW,EAAE,gBAAgB,EAAE,YAAY,EAAwB;IAErE,IAAI,MAAM,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,EAAE,sBAAsB,WAAW,EAAE,CAAC,CAAC;QAC1E,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAClD,QAAQ,CAAC,WAAW,CAAC;YACrB,QAAQ,CAAC,WAAW,CAAC;SACtB,CAAC,CAAC;QAEH,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAC3B,CAAC;IAED,MAAM,CAAC,KAAK,CACV,eAAe,YAAY,CAAC,IAAI,UAAU,YAAY,CAAC,EAAE,OAAO,WAAW,EAAE,CAC9E,CAAC;IACF,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,YAAY,GAAG,WAAW,GAAG,OAAO,CAAC;IAE3C,MAAM,cAAc,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IACtC,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC/D,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,KAAK,CACV,wBAAwB,YAAY,CAAC,IAAI,UAAU,YAAY,CAAC,EAAE,OAAO,YAAY,cAAc,eAAe,EAAE,CACrH,CAAC;QACF,cAAc,CAAC,KAAK,GAAG,SAAS,eAAe,GAAG,CAAC;IACrD,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,KAAK,CACpC,IAAI,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAC3D,CAAC;IACF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,kBAAkB,CAC1B,QAAQ,EACR,YAAY,CAAC,GAAG,EAChB,MAAM,QAAQ,CAAC,IAAI,EAAE,CACtB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,CAAC,YAAY,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;IAErD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC7E,MAAM,MAAM,CAAC,YAAY,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAEjD,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAClD,QAAQ,CAAC,YAAY,CAAC;QACtB,QAAQ,CAAC,YAAY,CAAC;KACvB,CAAC,CAAC;IAEH,MAAM,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IAExC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AAC3B,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,QAAgB;IACtC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAElC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,gBAAgB,CAAC,QAAQ,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,KAAe,CAAC,CAAC;IAE/B,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,IAAY,EAAE,QAAkB,EAAE,YAAoB;IACzE,OAAO,QAAQ,CACb,QAAQ,CAAC,IAAI,IAAI,EAAE,EACnB,iBAAiB,CAAC,IAAI,EAAE;QACtB,KAAK,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;KAC/D,CAAC,CACH,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"download.js","sourceRoot":"","sources":["../../../../src/downloader/asset/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAKhD,OAAO,EACL,MAAM,EACN,QAAQ,EACR,mBAAmB,EACnB,QAAQ,GACT,MAAM,qBAAqB,CAAC;AAgB7B,MAAM,kBAAmB,SAAQ,KAAK;IAMpC,YAAmB,QAAkB,EAAE,GAAW,EAAE,IAAa;QAC/D,KAAK,CAAC,0BAA0B,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;QACtC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,EAAE,OAAO,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,EAAqB,EACtD,EACE,WAAW,EACX,gBAAgB,EAChB,YAAY,EACZ,UAAU,GACW;IAEvB,IAAI,MAAM,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,EAAE,sBAAsB,WAAW,EAAE,CAAC,CAAC;QAC1E,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACzD,IAAI,CAAC,WAAW,CAAC;YACjB,QAAQ,CAAC,WAAW,CAAC;SACtB,CAAC,CAAC;QAEH,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,mBAAmB,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACrE,CAAC;IAED,MAAM,CAAC,KAAK,CACV,eAAe,YAAY,CAAC,IAAI,UAAU,YAAY,CAAC,EAAE,OAAO,WAAW,EAAE,CAC9E,CAAC;IACF,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,YAAY,GAAG,WAAW,GAAG,OAAO,CAAC;IAE3C,MAAM,cAAc,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IACtC,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC/D,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,KAAK,CACV,wBAAwB,YAAY,CAAC,IAAI,UAAU,YAAY,CAAC,EAAE,OAAO,YAAY,cAAc,eAAe,EAAE,CACrH,CAAC;QACF,cAAc,CAAC,KAAK,GAAG,SAAS,eAAe,GAAG,CAAC;IACrD,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,KAAK,CACpC,IAAI,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAC3D,CAAC;IACF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,kBAAkB,CAC1B,QAAQ,EACR,YAAY,CAAC,GAAG,EAChB,MAAM,QAAQ,CAAC,IAAI,EAAE,CACtB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,CAAC,YAAY,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;IAErD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,eAAe,CAC/B,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC,EAC1D,UAAU,CACX,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAEjD,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAClD,QAAQ,CAAC,YAAY,CAAC;QACtB,QAAQ,CAAC,YAAY,CAAC;KACvB,CAAC,CAAC;IAEH,MAAM,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IACxC,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC;IAE1C,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,mBAAmB,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;AACrE,CAAC;AAED,SAAS,eAAe,CAAC,SAAe,EAAE,UAAkB;IAC1D,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,UAAU,GAAG,IAAK,CAAC,CAAC;AAC5D,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,IAAY,EAAE,QAAkB,EAAE,YAAoB;IACzE,OAAO,QAAQ,CACb,QAAQ,CAAC,IAAI,IAAI,EAAE,EACnB,iBAAiB,CAAC,IAAI,EAAE;QACtB,KAAK,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;KAC/D,CAAC,CACH,CAAC;AACJ,CAAC"}
@@ -1,19 +1,22 @@
1
1
  import type { Logger } from "pino";
2
- export declare const DOWNLOAD_HELP = "Usage: fanbox-dl download [options]\n\nDownload FANBOX posts for selected creators.\n\nSelectors:\n --creator <id> Add a creator ID. Can be repeated.\n --following Add all followed creators.\n --supporting Add all supporting creators.\n --ignore-creator <id> Exclude a creator ID. Can be repeated.\n\nAuth:\n --cookie <value> Raw session ID or FANBOXSESSID=... cookie.\n --cookie-file <path> Read raw cookie or Netscape cookies.txt.\n --user-agent <value> Send the User-Agent from your logged-in browser.\n FANBOX_SESSION_ID Environment fallback.\n FANBOX_USER_AGENT User-Agent environment fallback.\n\nDownload:\n --output <path> Output directory. Default: fanbox-downloads.\n --flat-posts Store post files directly under each creator.\n\nRequests:\n --concurrency <n> Concurrent requests. Default: 10.\n --http2-session-multiplier <n>\n HTTP/2 sessions per origin multiplier based on concurrency. Default: 10.\n --request-interval-ms <n> Delay between request starts. Default: 500.\n --rate-limit-pause-ms <n> Force overwrite pause ms when 429.\n --max-retries <n> Retry attempts. Default: 3.\n\nOutput:\n --log-level <level> fatal|error|warn|info|debug|trace|silent. Default: info.\n --help Show this help.\n";
2
+ export declare const DOWNLOAD_HELP = "Usage: fanbox-dl download [options]\n\nDownload FANBOX posts for selected creators.\n\nSelectors:\n --creator <id> Add a creator ID. Can be repeated.\n --following Add all followed creators.\n --supporting Add all supporting creators.\n --ignore-creator <id> Exclude a creator ID. Can be repeated.\n\nAuth:\n --cookie <value> Raw session ID or FANBOXSESSID=... cookie.\n --cookie-file <path> Read raw cookie or Netscape cookies.txt.\n --user-agent <value> Send the User-Agent from your logged-in browser.\n FANBOX_SESSION_ID Environment fallback.\n\nDownload:\n --output <path> Output directory. Default: fanbox-downloads.\n --flat-posts Store post files directly under each creator.\n --max-filename-bytes <n> Max filename bytes including .part temp suffix. Default: 256.\n --flat-parent-min-bytes <n>\n Min optional parent/post name bytes to preserve in flat-posts filenames. Default: 35.\n --verify Verify skipped complete posts against local files.\n\nRequests:\n --concurrency <n> Concurrent requests. Default: 10.\n --http2-session-multiplier <n>\n HTTP/2 sessions per origin multiplier based on concurrency. Default: 10.\n --request-interval-ms <n> Delay between request starts. Default: 500.\n --rate-limit-pause-ms <n> Force overwrite pause ms when 429.\n --max-retries <n> Retry attempts. Default: 3.\n\nOutput:\n --log-level <level> fatal|error|warn|info|debug|trace|silent. Default: info.\n --help Show this help.\n";
3
3
  export interface DownloadOptions {
4
4
  concurrency: number;
5
5
  cookie?: string;
6
6
  creatorIds: string[];
7
+ flatParentMinBytes: number;
7
8
  flatPosts: boolean;
8
9
  following: boolean;
9
10
  http2SessionMultiplier: number;
10
11
  ignoreCreatorIds: string[];
12
+ maxFilenameBytes: number;
11
13
  maxRetries: number;
12
14
  output: string;
13
15
  rateLimitPauseMs?: number;
14
16
  requestIntervalMs: number;
15
17
  supporting: boolean;
16
18
  userAgent?: string;
19
+ verify: boolean;
17
20
  }
18
21
  interface ParseDownloadOptionsDeps {
19
22
  logger: Logger;
@@ -1 +1 @@
1
- {"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../../../src/downloader/cli/options.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAKnC,eAAO,MAAM,aAAa,03CAgCzB,CAAC;AAEF,MAAM,WAAW,eAAe;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,OAAO,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,wBAAwB;IAChC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,oBAAoB,CAClC,EAAE,MAAM,EAAE,EAAE,wBAAwB,EACpC,IAAI,EAAE,MAAM,EAAE,GACb,eAAe,CAmCjB"}
1
+ {"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../../../src/downloader/cli/options.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAKnC,eAAO,MAAM,aAAa,ynDAmCzB,CAAC;AAEF,MAAM,WAAW,eAAe;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,OAAO,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,UAAU,wBAAwB;IAChC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,oBAAoB,CAClC,EAAE,MAAM,EAAE,EAAE,wBAAwB,EACpC,IAAI,EAAE,MAAM,EAAE,GACb,eAAe,CA4CjB"}
@@ -17,11 +17,14 @@ Auth:
17
17
  --cookie-file <path> Read raw cookie or Netscape cookies.txt.
18
18
  --user-agent <value> Send the User-Agent from your logged-in browser.
19
19
  FANBOX_SESSION_ID Environment fallback.
20
- FANBOX_USER_AGENT User-Agent environment fallback.
21
20
 
22
21
  Download:
23
22
  --output <path> Output directory. Default: fanbox-downloads.
24
23
  --flat-posts Store post files directly under each creator.
24
+ --max-filename-bytes <n> Max filename bytes including .part temp suffix. Default: 256.
25
+ --flat-parent-min-bytes <n>
26
+ Min optional parent/post name bytes to preserve in flat-posts filenames. Default: 35.
27
+ --verify Verify skipped complete posts against local files.
25
28
 
26
29
  Requests:
27
30
  --concurrency <n> Concurrent requests. Default: 10.
@@ -46,10 +49,12 @@ export function parseDownloadOptions({ logger }, args) {
46
49
  concurrency: parsePositiveInteger("concurrency", values.concurrency),
47
50
  cookie,
48
51
  creatorIds,
52
+ flatParentMinBytes: parsePositiveInteger("flat-parent-min-bytes", values["flat-parent-min-bytes"]),
49
53
  flatPosts: values["flat-posts"],
50
54
  following: values.following,
51
55
  http2SessionMultiplier: parsePositiveInteger("http2-session-multiplier", values["http2-session-multiplier"]),
52
56
  ignoreCreatorIds: values["ignore-creator"] ?? [],
57
+ maxFilenameBytes: parsePositiveInteger("max-filename-bytes", values["max-filename-bytes"]),
53
58
  maxRetries: parseNonNegativeInteger("max-retries", values["max-retries"]),
54
59
  output: values.output,
55
60
  rateLimitPauseMs: values["rate-limit-pause-ms"]
@@ -58,6 +63,7 @@ export function parseDownloadOptions({ logger }, args) {
58
63
  requestIntervalMs: parseNonNegativeInteger("request-interval-ms", values["request-interval-ms"]),
59
64
  supporting: values.supporting,
60
65
  userAgent: values["user-agent"],
66
+ verify: values.verify,
61
67
  };
62
68
  }
63
69
  function parseCookie({ logger }, values) {
@@ -88,17 +94,20 @@ function parseDownloadArgs(args) {
88
94
  "cookie-file": { type: "string" },
89
95
  creator: { multiple: true, type: "string" },
90
96
  "dry-run": { default: false, type: "boolean" },
97
+ "flat-parent-min-bytes": { default: "35", type: "string" },
91
98
  "flat-posts": { default: false, type: "boolean" },
92
99
  following: { default: false, type: "boolean" },
93
100
  "http2-session-multiplier": { default: "10", type: "string" },
94
101
  "ignore-creator": { multiple: true, type: "string" },
95
102
  "log-level": { default: "info", type: "string" },
103
+ "max-filename-bytes": { default: "256", type: "string" },
96
104
  "max-retries": { default: "3", type: "string" },
97
105
  output: { default: "fanbox-downloads", type: "string" },
98
106
  "rate-limit-pause-ms": { type: "string" },
99
107
  "request-interval-ms": { default: "500", type: "string" },
100
108
  supporting: { default: false, type: "boolean" },
101
109
  "user-agent": { type: "string" },
110
+ verify: { default: false, type: "boolean" },
102
111
  },
103
112
  strict: true,
104
113
  });
@@ -1 +1 @@
1
- {"version":3,"file":"options.js","sourceRoot":"","sources":["../../../../src/downloader/cli/options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAItC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C,MAAM,CAAC,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgC5B,CAAC;AAsBF,MAAM,UAAU,oBAAoB,CAClC,EAAE,MAAM,EAA4B,EACpC,IAAc;IAEd,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,EAAE,0BAA0B,CAAC,CAAC;IACnD,MAAM,EAAE,MAAM,EAAE,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IACxC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,UAAU;QACpE,MAAM,IAAI,aAAa,CAAC,2CAA2C,CAAC,CAAC;IAEvE,MAAM,MAAM,GAAG,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,CAAC;IAE/C,OAAO;QACL,WAAW,EAAE,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,WAAW,CAAC;QACpE,MAAM;QACN,UAAU;QACV,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC;QAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,sBAAsB,EAAE,oBAAoB,CAC1C,0BAA0B,EAC1B,MAAM,CAAC,0BAA0B,CAAC,CACnC;QACD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;QAChD,UAAU,EAAE,uBAAuB,CAAC,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QACzE,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,gBAAgB,EAAE,MAAM,CAAC,qBAAqB,CAAC;YAC7C,CAAC,CAAC,uBAAuB,CACrB,qBAAqB,EACrB,MAAM,CAAC,qBAAqB,CAAC,CAC9B;YACH,CAAC,CAAC,SAAS;QACb,iBAAiB,EAAE,uBAAuB,CACxC,qBAAqB,EACrB,MAAM,CAAC,qBAAqB,CAAC,CAC9B;QACD,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC;KAChC,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAClB,EAAE,MAAM,EAA4B,EACpC,MAAsD;IAEtD,MAAM,UAAU,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,EAAE,8BAA8B,CAAC,CAAC;QACxE,OAAO,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,EAAE,wBAAwB,CAAC,CAAC;QACvD,OAAO,eAAe,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;QAClC,MAAM,CAAC,KAAK,CACV,EAAE,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,EACpD,+BAA+B,CAChC,CAAC;QACF,OAAO,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACnC,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAc;IACvC,IAAI,CAAC;QACH,OAAO,SAAS,CAAC;YACf,gBAAgB,EAAE,KAAK;YACvB,IAAI;YACJ,OAAO,EAAE;gBACP,WAAW,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC9C,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC1B,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACjC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC3C,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC9C,YAAY,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBACjD,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC9C,0BAA0B,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC7D,gBAAgB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACpD,WAAW,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAChD,aAAa,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC/C,MAAM,EAAE,EAAE,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACvD,qBAAqB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzC,qBAAqB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzD,UAAU,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC/C,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aACjC;YACD,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,aAAa,CAAE,KAAe,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,IAAY,EAAE,KAAa;IAC1D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,aAAa,CAAC,GAAG,IAAI,iCAAiC,CAAC,CAAC;IACpE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY,EAAE,KAAa;IACvD,MAAM,MAAM,GAAG,uBAAuB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACpD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,MAAM,IAAI,aAAa,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
1
+ {"version":3,"file":"options.js","sourceRoot":"","sources":["../../../../src/downloader/cli/options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAItC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C,MAAM,CAAC,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmC5B,CAAC;AAyBF,MAAM,UAAU,oBAAoB,CAClC,EAAE,MAAM,EAA4B,EACpC,IAAc;IAEd,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,EAAE,0BAA0B,CAAC,CAAC;IACnD,MAAM,EAAE,MAAM,EAAE,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IACxC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,UAAU;QACpE,MAAM,IAAI,aAAa,CAAC,2CAA2C,CAAC,CAAC;IAEvE,MAAM,MAAM,GAAG,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,CAAC;IAE/C,OAAO;QACL,WAAW,EAAE,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,WAAW,CAAC;QACpE,MAAM;QACN,UAAU;QACV,kBAAkB,EAAE,oBAAoB,CACtC,uBAAuB,EACvB,MAAM,CAAC,uBAAuB,CAAC,CAChC;QACD,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC;QAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,sBAAsB,EAAE,oBAAoB,CAC1C,0BAA0B,EAC1B,MAAM,CAAC,0BAA0B,CAAC,CACnC;QACD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;QAChD,gBAAgB,EAAE,oBAAoB,CACpC,oBAAoB,EACpB,MAAM,CAAC,oBAAoB,CAAC,CAC7B;QACD,UAAU,EAAE,uBAAuB,CAAC,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QACzE,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,gBAAgB,EAAE,MAAM,CAAC,qBAAqB,CAAC;YAC7C,CAAC,CAAC,uBAAuB,CACrB,qBAAqB,EACrB,MAAM,CAAC,qBAAqB,CAAC,CAC9B;YACH,CAAC,CAAC,SAAS;QACb,iBAAiB,EAAE,uBAAuB,CACxC,qBAAqB,EACrB,MAAM,CAAC,qBAAqB,CAAC,CAC9B;QACD,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;KACtB,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAClB,EAAE,MAAM,EAA4B,EACpC,MAAsD;IAEtD,MAAM,UAAU,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,EAAE,8BAA8B,CAAC,CAAC;QACxE,OAAO,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,EAAE,wBAAwB,CAAC,CAAC;QACvD,OAAO,eAAe,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;QAClC,MAAM,CAAC,KAAK,CACV,EAAE,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,EACpD,+BAA+B,CAChC,CAAC;QACF,OAAO,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACnC,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAc;IACvC,IAAI,CAAC;QACH,OAAO,SAAS,CAAC;YACf,gBAAgB,EAAE,KAAK;YACvB,IAAI;YACJ,OAAO,EAAE;gBACP,WAAW,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC9C,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC1B,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACjC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC3C,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC9C,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC1D,YAAY,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBACjD,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC9C,0BAA0B,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC7D,gBAAgB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACpD,WAAW,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAChD,oBAAoB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACxD,aAAa,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC/C,MAAM,EAAE,EAAE,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACvD,qBAAqB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzC,qBAAqB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzD,UAAU,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC/C,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAChC,MAAM,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;aAC5C;YACD,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,aAAa,CAAE,KAAe,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,IAAY,EAAE,KAAa;IAC1D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,aAAa,CAAC,GAAG,IAAI,iCAAiC,CAAC,CAAC;IACpE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY,EAAE,KAAa;IACvD,MAAM,MAAM,GAAG,uBAAuB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACpD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,MAAM,IAAI,aAAa,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"discover-posts.d.ts","sourceRoot":"","sources":["../../../../src/downloader/creator/discover-posts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAEV,WAAW,EACZ,MAAM,uBAAuB,CAAC;AAE/B,UAAU,gCAAgC;IACxC,MAAM,EAAE,YAAY,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,2BAA2B;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,gCAAgC,EACpD,EACE,SAAS,EACT,OAAO,EACP,sBAAsB,EACtB,KAAW,GACZ,EAAE,2BAA2B,GAC7B,OAAO,CAAC,WAAW,EAAE,CAAC,CA+BxB"}
1
+ {"version":3,"file":"discover-posts.d.ts","sourceRoot":"","sources":["../../../../src/downloader/creator/discover-posts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAEV,WAAW,EACZ,MAAM,uBAAuB,CAAC;AAE/B,UAAU,gCAAgC;IACxC,MAAM,EAAE,YAAY,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,2BAA2B;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,gCAAgC,EACpD,EACE,SAAS,EACT,OAAO,EACP,sBAAsB,EACtB,KAAW,GACZ,EAAE,2BAA2B,GAC7B,OAAO,CAAC,WAAW,EAAE,CAAC,CA6BxB"}
@@ -11,11 +11,13 @@ export async function discoverCreatorPosts({ client, logger }, { creatorId, firs
11
11
  logger.debug(`Discovered ${posts.length} posts for creator ${creatorId}`);
12
12
  if (posts.length < limit)
13
13
  return posts;
14
- const last = posts.at(-1);
14
+ const lastPost = posts.at(-1);
15
+ if (!lastPost)
16
+ return posts;
15
17
  const remains = await discoverCreatorPosts({ client, logger }, {
16
18
  creatorId,
17
- firstId: last.id,
18
- firstPublishedDatetime: last.publishedDatetime,
19
+ firstId: lastPost.id,
20
+ firstPublishedDatetime: lastPost.publishedDatetime,
19
21
  limit,
20
22
  });
21
23
  return [...posts, ...remains.slice(1)];
@@ -1 +1 @@
1
- {"version":3,"file":"discover-posts.js","sourceRoot":"","sources":["../../../../src/downloader/creator/discover-posts.ts"],"names":[],"mappings":"AAoBA,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAoC,EACpD,EACE,SAAS,EACT,OAAO,EACP,sBAAsB,EACtB,KAAK,GAAG,GAAG,GACiB;IAE9B,MAAM,CAAC,KAAK,CACV,mBAAmB,KAAK,eAAe,OAAO,IAAI,OAAO,gBAAgB,SAAS,EAAE,CACrF,CAAC;IACF,MAAM,uBAAuB,GAAG;QAC9B,SAAS;QACT,OAAO;QACP,sBAAsB;QACtB,KAAK;QACL,IAAI,EAAE,QAAQ;KACkB,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;IACrE,MAAM,CAAC,KAAK,CAAC,cAAc,KAAK,CAAC,MAAM,sBAAsB,SAAS,EAAE,CAAC,CAAC;IAC1E,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK;QAAE,OAAO,KAAK,CAAC;IAIvC,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC;IAG3B,MAAM,OAAO,GAAG,MAAM,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAE,EAClB;QACE,SAAS;QACT,OAAO,EAAE,IAAI,CAAC,EAAE;QAChB,sBAAsB,EAAE,IAAI,CAAC,iBAAiB;QAC9C,KAAK;KACN,CACF,CAAC;IAEF,OAAO,CAAC,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC"}
1
+ {"version":3,"file":"discover-posts.js","sourceRoot":"","sources":["../../../../src/downloader/creator/discover-posts.ts"],"names":[],"mappings":"AAoBA,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAoC,EACpD,EACE,SAAS,EACT,OAAO,EACP,sBAAsB,EACtB,KAAK,GAAG,GAAG,GACiB;IAE9B,MAAM,CAAC,KAAK,CACV,mBAAmB,KAAK,eAAe,OAAO,IAAI,OAAO,gBAAgB,SAAS,EAAE,CACrF,CAAC;IACF,MAAM,uBAAuB,GAAG;QAC9B,SAAS;QACT,OAAO;QACP,sBAAsB;QACtB,KAAK;QACL,IAAI,EAAE,QAAQ;KACkB,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;IACrE,MAAM,CAAC,KAAK,CAAC,cAAc,KAAK,CAAC,MAAM,sBAAsB,SAAS,EAAE,CAAC,CAAC;IAC1E,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK;QAAE,OAAO,KAAK,CAAC;IAEvC,MAAM,QAAQ,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAE5B,MAAM,OAAO,GAAG,MAAM,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAE,EAClB;QACE,SAAS;QACT,OAAO,EAAE,QAAQ,CAAC,EAAE;QACpB,sBAAsB,EAAE,QAAQ,CAAC,iBAAiB;QAClD,KAAK;KACN,CACF,CAAC;IAEF,OAAO,CAAC,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC"}