@moontaiworks/fanbox-dl 1.5.5 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -37
- package/dist/cjs/downloader/asset/download.d.ts +3 -1
- package/dist/cjs/downloader/asset/download.d.ts.map +1 -1
- package/dist/cjs/downloader/asset/download.js +11 -14
- package/dist/cjs/downloader/asset/download.js.map +1 -1
- package/dist/cjs/downloader/cli/options.d.ts +4 -1
- package/dist/cjs/downloader/cli/options.d.ts.map +1 -1
- package/dist/cjs/downloader/cli/options.js +10 -1
- package/dist/cjs/downloader/cli/options.js.map +1 -1
- package/dist/cjs/downloader/creator/discover-posts.d.ts.map +1 -1
- package/dist/cjs/downloader/creator/discover-posts.js +5 -3
- package/dist/cjs/downloader/creator/discover-posts.js.map +1 -1
- package/dist/cjs/downloader/creator/sync.d.ts +2 -1
- package/dist/cjs/downloader/creator/sync.d.ts.map +1 -1
- package/dist/cjs/downloader/creator/sync.js +26 -27
- package/dist/cjs/downloader/creator/sync.js.map +1 -1
- package/dist/cjs/downloader/fs/filesystem.d.ts +3 -0
- package/dist/cjs/downloader/fs/filesystem.d.ts.map +1 -1
- package/dist/cjs/downloader/fs/filesystem.js +27 -2
- package/dist/cjs/downloader/fs/filesystem.js.map +1 -1
- package/dist/cjs/downloader/fs/path-manager.d.ts +10 -3
- package/dist/cjs/downloader/fs/path-manager.d.ts.map +1 -1
- package/dist/cjs/downloader/fs/path-manager.js +102 -12
- package/dist/cjs/downloader/fs/path-manager.js.map +1 -1
- package/dist/cjs/downloader/index.d.ts.map +1 -1
- package/dist/cjs/downloader/index.js +16 -13
- package/dist/cjs/downloader/index.js.map +1 -1
- package/dist/cjs/downloader/manifest/creator-manager.d.ts +3 -0
- package/dist/cjs/downloader/manifest/creator-manager.d.ts.map +1 -1
- package/dist/cjs/downloader/manifest/creator-manager.js +30 -2
- package/dist/cjs/downloader/manifest/creator-manager.js.map +1 -1
- package/dist/cjs/downloader/manifest/creator.d.ts +6 -0
- package/dist/cjs/downloader/manifest/creator.d.ts.map +1 -1
- package/dist/cjs/downloader/manifest/creator.js +11 -1
- package/dist/cjs/downloader/manifest/creator.js.map +1 -1
- package/dist/cjs/downloader/post/article.d.ts.map +1 -1
- package/dist/cjs/downloader/post/article.js +25 -22
- package/dist/cjs/downloader/post/article.js.map +1 -1
- package/dist/cjs/downloader/post/sync.d.ts +3 -2
- package/dist/cjs/downloader/post/sync.d.ts.map +1 -1
- package/dist/cjs/downloader/post/sync.js +127 -79
- package/dist/cjs/downloader/post/sync.js.map +1 -1
- package/dist/cjs/downloader/post/verify.d.ts +9 -0
- package/dist/cjs/downloader/post/verify.d.ts.map +1 -0
- package/dist/cjs/downloader/post/verify.js +64 -0
- package/dist/cjs/downloader/post/verify.js.map +1 -0
- package/dist/esm/downloader/asset/download.d.ts +3 -1
- package/dist/esm/downloader/asset/download.d.ts.map +1 -1
- package/dist/esm/downloader/asset/download.js +12 -15
- package/dist/esm/downloader/asset/download.js.map +1 -1
- package/dist/esm/downloader/cli/options.d.ts +4 -1
- package/dist/esm/downloader/cli/options.d.ts.map +1 -1
- package/dist/esm/downloader/cli/options.js +10 -1
- package/dist/esm/downloader/cli/options.js.map +1 -1
- package/dist/esm/downloader/creator/discover-posts.d.ts.map +1 -1
- package/dist/esm/downloader/creator/discover-posts.js +5 -3
- package/dist/esm/downloader/creator/discover-posts.js.map +1 -1
- package/dist/esm/downloader/creator/sync.d.ts +2 -1
- package/dist/esm/downloader/creator/sync.d.ts.map +1 -1
- package/dist/esm/downloader/creator/sync.js +26 -27
- package/dist/esm/downloader/creator/sync.js.map +1 -1
- package/dist/esm/downloader/fs/filesystem.d.ts +3 -0
- package/dist/esm/downloader/fs/filesystem.d.ts.map +1 -1
- package/dist/esm/downloader/fs/filesystem.js +24 -2
- package/dist/esm/downloader/fs/filesystem.js.map +1 -1
- package/dist/esm/downloader/fs/path-manager.d.ts +10 -3
- package/dist/esm/downloader/fs/path-manager.d.ts.map +1 -1
- package/dist/esm/downloader/fs/path-manager.js +102 -12
- package/dist/esm/downloader/fs/path-manager.js.map +1 -1
- package/dist/esm/downloader/index.d.ts.map +1 -1
- package/dist/esm/downloader/index.js +16 -13
- package/dist/esm/downloader/index.js.map +1 -1
- package/dist/esm/downloader/manifest/creator-manager.d.ts +3 -0
- package/dist/esm/downloader/manifest/creator-manager.d.ts.map +1 -1
- package/dist/esm/downloader/manifest/creator-manager.js +30 -2
- package/dist/esm/downloader/manifest/creator-manager.js.map +1 -1
- package/dist/esm/downloader/manifest/creator.d.ts +6 -0
- package/dist/esm/downloader/manifest/creator.d.ts.map +1 -1
- package/dist/esm/downloader/manifest/creator.js +11 -1
- package/dist/esm/downloader/manifest/creator.js.map +1 -1
- package/dist/esm/downloader/post/article.d.ts.map +1 -1
- package/dist/esm/downloader/post/article.js +25 -22
- package/dist/esm/downloader/post/article.js.map +1 -1
- package/dist/esm/downloader/post/sync.d.ts +3 -2
- package/dist/esm/downloader/post/sync.d.ts.map +1 -1
- package/dist/esm/downloader/post/sync.js +127 -79
- package/dist/esm/downloader/post/sync.js.map +1 -1
- package/dist/esm/downloader/post/verify.d.ts +9 -0
- package/dist/esm/downloader/post/verify.d.ts.map +1 -0
- package/dist/esm/downloader/post/verify.js +61 -0
- package/dist/esm/downloader/post/verify.js.map +1 -0
- package/package.json +1 -1
|
@@ -8,7 +8,17 @@ const download_js_1 = require("../asset/download.js");
|
|
|
8
8
|
const asset_js_1 = require("../markdown/asset.js");
|
|
9
9
|
const content_js_1 = require("./content.js");
|
|
10
10
|
const contents_js_1 = require("./contents.js");
|
|
11
|
-
|
|
11
|
+
const verify_js_1 = require("./verify.js");
|
|
12
|
+
async function preSyncPostCheck({ logger, manifest, verify }, postSummary) {
|
|
13
|
+
const existingPost = manifest.posts[postSummary.id];
|
|
14
|
+
if (existingPost?.status === "complete" &&
|
|
15
|
+
postSummary.updatedDatetime === existingPost.updatedDatetime) {
|
|
16
|
+
logger.debug(`Post ${postSummary.id} has no changes, skipping download.`);
|
|
17
|
+
if (verify) {
|
|
18
|
+
return (0, verify_js_1.verifyCompletePost)({ logger }, existingPost, postSummary);
|
|
19
|
+
}
|
|
20
|
+
return existingPost;
|
|
21
|
+
}
|
|
12
22
|
if (postSummary.isRestricted) {
|
|
13
23
|
logger.debug(`Post ${postSummary.id} is restricted, skipping download.`);
|
|
14
24
|
return {
|
|
@@ -19,12 +29,6 @@ function preSyncPostCheck({ logger, manifest }, postSummary) {
|
|
|
19
29
|
updatedDatetime: postSummary.updatedDatetime,
|
|
20
30
|
};
|
|
21
31
|
}
|
|
22
|
-
const existingPost = manifest.posts[postSummary.id];
|
|
23
|
-
if (existingPost?.status === "complete" &&
|
|
24
|
-
postSummary.updatedDatetime === existingPost.updatedDatetime) {
|
|
25
|
-
logger.debug(`Post ${postSummary.id} has no changes, skipping download.`);
|
|
26
|
-
return existingPost;
|
|
27
|
-
}
|
|
28
32
|
return {
|
|
29
33
|
assets: {},
|
|
30
34
|
id: postSummary.id,
|
|
@@ -38,86 +42,31 @@ async function syncPost({ headers, logger, pathManager, transport }, post) {
|
|
|
38
42
|
const contents = (0, contents_js_1.formatPostContents)({ logger }, post);
|
|
39
43
|
if (post.coverImageUrl)
|
|
40
44
|
contents.unshift(formatCoverImage(post));
|
|
41
|
-
let hasUnknownContent = false;
|
|
42
|
-
const assetManifestData = {};
|
|
43
|
-
const download = download_js_1.downloadAsset.bind(undefined, {
|
|
44
|
-
headers,
|
|
45
|
-
logger,
|
|
46
|
-
transport,
|
|
47
|
-
});
|
|
48
45
|
const totalDigits = contents.length.toString().length;
|
|
49
|
-
const results = await Promise.allSettled(contents.map(
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
path: destination,
|
|
61
|
-
status: "failed",
|
|
62
|
-
url: content.url,
|
|
63
|
-
};
|
|
64
|
-
throw err;
|
|
65
|
-
});
|
|
66
|
-
assetManifestData[content.id] = {
|
|
67
|
-
bytes,
|
|
68
|
-
path: destination,
|
|
69
|
-
sha256,
|
|
70
|
-
status: "complete",
|
|
71
|
-
url: content.url,
|
|
72
|
-
};
|
|
73
|
-
return (0, asset_js_1.formatImageAsset)({ assetPath: destination });
|
|
74
|
-
}
|
|
75
|
-
if (content instanceof content_js_1.FileContent) {
|
|
76
|
-
const destination = pathManager.asset(indexPadded, `${content.name}-${content.id}`, content.extension);
|
|
77
|
-
const { bytes, sha256 } = await download({
|
|
78
|
-
destination,
|
|
79
|
-
fallbackDateTime: post.updatedDatetime,
|
|
80
|
-
mediaContent: content,
|
|
81
|
-
}).catch((err) => {
|
|
82
|
-
logger.error({ err }, `Error occurred while downloading file asset ${content.id}, skipping.`);
|
|
83
|
-
assetManifestData[content.id] = {
|
|
84
|
-
path: destination,
|
|
85
|
-
status: "failed",
|
|
86
|
-
url: content.url,
|
|
87
|
-
};
|
|
88
|
-
throw err;
|
|
89
|
-
});
|
|
90
|
-
assetManifestData[content.id] = {
|
|
91
|
-
bytes,
|
|
92
|
-
path: destination,
|
|
93
|
-
sha256,
|
|
94
|
-
status: "complete",
|
|
95
|
-
url: content.url,
|
|
96
|
-
};
|
|
97
|
-
return (0, asset_js_1.formatFileAsset)({ assetPath: destination });
|
|
98
|
-
}
|
|
99
|
-
if (content instanceof content_js_1.TextContent) {
|
|
100
|
-
return (0, asset_js_1.formatTextContent)(content);
|
|
101
|
-
}
|
|
102
|
-
logger.warn(`Post ${post.id} has unhandled content type ${content.type}, skipping.`);
|
|
103
|
-
hasUnknownContent = true;
|
|
104
|
-
}));
|
|
105
|
-
const markdownContent = results.map((result) => {
|
|
106
|
-
if (result.status === "fulfilled") {
|
|
107
|
-
return result.value;
|
|
108
|
-
}
|
|
109
|
-
const err = String(result.reason);
|
|
110
|
-
return `<!-- Error occurred while processing content: ${err} -->`;
|
|
111
|
-
});
|
|
112
|
-
await (0, promises_1.writeFile)(pathManager.asset("0".padStart(totalDigits, "0"), "content", "md"), markdownContent.join("\n\n"));
|
|
46
|
+
const results = await Promise.allSettled(contents.map((content, index) => processContent({ headers, logger, pathManager, post, transport }, { content, index, totalDigits })));
|
|
47
|
+
const hasRejectedContent = results.some((result) => result.status === "rejected");
|
|
48
|
+
const fulfilled = fulfilledResults(results);
|
|
49
|
+
const hasUnknownContent = fulfilled.some((result) => result.unknown);
|
|
50
|
+
const hasFailedContent = fulfilled.some((result) => result.failed);
|
|
51
|
+
const assetManifestData = collectAssetManifestData(fulfilled);
|
|
52
|
+
const markdownContent = formatMarkdownContent(results);
|
|
53
|
+
await (0, promises_1.writeFile)(pathManager.asset([
|
|
54
|
+
{ context: "0".padStart(totalDigits, "0"), required: true },
|
|
55
|
+
{ context: "content", required: true },
|
|
56
|
+
], "md"), markdownContent.join("\n\n"));
|
|
113
57
|
return {
|
|
114
58
|
assets: assetManifestData,
|
|
115
59
|
id: post.id,
|
|
116
60
|
restricted: false,
|
|
117
|
-
status: hasUnknownContent
|
|
61
|
+
status: hasUnknownContent || hasFailedContent || hasRejectedContent
|
|
62
|
+
? "partial"
|
|
63
|
+
: "complete",
|
|
118
64
|
updatedDatetime: post.updatedDatetime,
|
|
119
65
|
};
|
|
120
66
|
}
|
|
67
|
+
function collectAssetManifestData(contents) {
|
|
68
|
+
return Object.fromEntries(contents.flatMap((content) => content.assets ?? []));
|
|
69
|
+
}
|
|
121
70
|
function formatCoverImage(post) {
|
|
122
71
|
if (!post.coverImageUrl)
|
|
123
72
|
throw new Error("Post does not have a cover image");
|
|
@@ -128,4 +77,103 @@ function formatCoverImage(post) {
|
|
|
128
77
|
originalUrl: post.coverImageUrl,
|
|
129
78
|
});
|
|
130
79
|
}
|
|
80
|
+
function formatMarkdownContent(results) {
|
|
81
|
+
return results.map((result) => {
|
|
82
|
+
if (result.status === "fulfilled") {
|
|
83
|
+
return result.value.markdown;
|
|
84
|
+
}
|
|
85
|
+
const err = String(result.reason);
|
|
86
|
+
return `<!-- Error occurred while processing content: ${err} -->`;
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
function formatMediaAsset(content, destination) {
|
|
90
|
+
if (content instanceof content_js_1.ImageContent) {
|
|
91
|
+
return (0, asset_js_1.formatImageAsset)({ assetPath: destination });
|
|
92
|
+
}
|
|
93
|
+
return (0, asset_js_1.formatFileAsset)({ assetPath: destination });
|
|
94
|
+
}
|
|
95
|
+
function fulfilledResults(results) {
|
|
96
|
+
return results
|
|
97
|
+
.filter((result) => result.status === "fulfilled")
|
|
98
|
+
.map((result) => result.value);
|
|
99
|
+
}
|
|
100
|
+
function isMediaContent(content) {
|
|
101
|
+
return content instanceof content_js_1.FileContent || content instanceof content_js_1.ImageContent;
|
|
102
|
+
}
|
|
103
|
+
function mediaAssetSegments(content, index, totalDigits) {
|
|
104
|
+
const indexPadded = index.toString().padStart(totalDigits, "0");
|
|
105
|
+
if (content instanceof content_js_1.FileContent) {
|
|
106
|
+
return [
|
|
107
|
+
{ context: indexPadded, required: true },
|
|
108
|
+
{ context: content.name, required: false },
|
|
109
|
+
{ context: content.id, required: true },
|
|
110
|
+
];
|
|
111
|
+
}
|
|
112
|
+
return [
|
|
113
|
+
{ context: indexPadded, required: true },
|
|
114
|
+
{ context: content.id, required: true },
|
|
115
|
+
];
|
|
116
|
+
}
|
|
117
|
+
async function processContent({ headers, logger, pathManager, post, transport }, { content, index, totalDigits }) {
|
|
118
|
+
if (isMediaContent(content)) {
|
|
119
|
+
return syncMediaContent({ headers, logger, pathManager, transport }, {
|
|
120
|
+
content,
|
|
121
|
+
fallbackDateTime: post.updatedDatetime,
|
|
122
|
+
index,
|
|
123
|
+
totalDigits,
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
if (content instanceof content_js_1.TextContent) {
|
|
127
|
+
return { markdown: (0, asset_js_1.formatTextContent)(content) };
|
|
128
|
+
}
|
|
129
|
+
logger.warn(`Post ${post.id} has unhandled content type ${content.type}, skipping.`);
|
|
130
|
+
return { unknown: true };
|
|
131
|
+
}
|
|
132
|
+
async function syncMediaContent({ headers, logger, pathManager, transport, }, { content, fallbackDateTime, index, totalDigits, }) {
|
|
133
|
+
const destination = pathManager.asset(mediaAssetSegments(content, index, totalDigits), content.extension);
|
|
134
|
+
const downloadResult = await (0, download_js_1.downloadAsset)({ headers, logger, transport }, {
|
|
135
|
+
destination,
|
|
136
|
+
fallbackDateTime,
|
|
137
|
+
mediaContent: content,
|
|
138
|
+
timeOffset: index,
|
|
139
|
+
})
|
|
140
|
+
.then(({ bytes, modifiedTime, sha256 }) => ({ bytes, failed: false, modifiedTime, sha256 }))
|
|
141
|
+
.catch((err) => {
|
|
142
|
+
logger.error({ err }, `Error occurred while downloading ${content.type} asset ${content.id}, skipping.`);
|
|
143
|
+
return { error: err, failed: true };
|
|
144
|
+
});
|
|
145
|
+
if (downloadResult.failed) {
|
|
146
|
+
return {
|
|
147
|
+
assets: [
|
|
148
|
+
[
|
|
149
|
+
content.id,
|
|
150
|
+
{
|
|
151
|
+
path: destination,
|
|
152
|
+
status: "failed",
|
|
153
|
+
url: content.url,
|
|
154
|
+
},
|
|
155
|
+
],
|
|
156
|
+
],
|
|
157
|
+
failed: true,
|
|
158
|
+
markdown: `<!-- Error occurred while processing content: ${String(downloadResult.error)} -->`,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
return {
|
|
162
|
+
assets: [
|
|
163
|
+
[
|
|
164
|
+
content.id,
|
|
165
|
+
{
|
|
166
|
+
bytes: downloadResult.bytes,
|
|
167
|
+
contentIndex: index,
|
|
168
|
+
expectedTime: downloadResult.modifiedTime,
|
|
169
|
+
path: destination,
|
|
170
|
+
sha256: downloadResult.sha256,
|
|
171
|
+
status: "complete",
|
|
172
|
+
url: content.url,
|
|
173
|
+
},
|
|
174
|
+
],
|
|
175
|
+
],
|
|
176
|
+
markdown: formatMediaAsset(content, destination),
|
|
177
|
+
};
|
|
178
|
+
}
|
|
131
179
|
//# sourceMappingURL=sync.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../../../src/downloader/post/sync.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../../../src/downloader/post/sync.ts"],"names":[],"mappings":";;AAiEA,4CAqCC;AAED,4BAiDC;AAzJD,+CAA6C;AAC7C,yCAAoC;AAMpC,sDAAqD;AAOrD,mDAI8B;AAC9B,6CAAsE;AAEtE,+CAAmD;AACnD,2CAAiD;AA2C1C,KAAK,UAAU,gBAAgB,CACpC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAwB,EAClD,WAAwB;IAExB,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IACpD,IACE,YAAY,EAAE,MAAM,KAAK,UAAU;QACnC,WAAW,CAAC,eAAe,KAAK,YAAY,CAAC,eAAe,EAC5D,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,QAAQ,WAAW,CAAC,EAAE,qCAAqC,CAAC,CAAC;QAC1E,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,IAAA,8BAAkB,EAAC,EAAE,MAAM,EAAE,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;QACnE,CAAC;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,IAAI,WAAW,CAAC,YAAY,EAAE,CAAC;QAE7B,MAAM,CAAC,KAAK,CAAC,QAAQ,WAAW,CAAC,EAAE,oCAAoC,CAAC,CAAC;QACzE,OAAO;YACL,MAAM,EAAE,EAAE;YACV,EAAE,EAAE,WAAW,CAAC,EAAE;YAClB,UAAU,EAAE,IAAI;YAChB,MAAM,EAAE,SAAS;YACjB,eAAe,EAAE,WAAW,CAAC,eAAe;SAC7C,CAAC;IACJ,CAAC;IAED,OAAO;QACL,MAAM,EAAE,EAAE;QACV,EAAE,EAAE,WAAW,CAAC,EAAE;QAClB,UAAU,EAAE,KAAK;QACjB,MAAM,EAAE,SAAS;QACjB,eAAe,EAAE,WAAW,CAAC,eAAe;KAC7C,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,QAAQ,CAC5B,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAgB,EACzD,IAAU;IAEV,MAAM,CAAC,IAAI,CACT,oBAAoB,IAAI,CAAC,EAAE,eAAe,IAAI,CAAC,eAAe,eAAe,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,KAAK,EAAE,CAC7G,CAAC;IACF,MAAM,QAAQ,GAAG,IAAA,gCAAkB,EAAC,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,CAAC;IACtD,IAAI,IAAI,CAAC,aAAa;QAAE,QAAQ,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;IAEjE,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC;IACtD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAC9B,cAAc,CACZ,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,EACjD,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,CAChC,CACF,CACF,CAAC;IACF,MAAM,kBAAkB,GAAG,OAAO,CAAC,IAAI,CACrC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,UAAU,CACzC,CAAC;IACF,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,iBAAiB,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACrE,MAAM,gBAAgB,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACnE,MAAM,iBAAiB,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC;IAC9D,MAAM,eAAe,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;IAEvD,MAAM,IAAA,oBAAS,EACb,WAAW,CAAC,KAAK,CACf;QACE,EAAE,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE;QAC3D,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE;KACvC,EACD,IAAI,CACL,EACD,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAC7B,CAAC;IAEF,OAAO;QACL,MAAM,EAAE,iBAAiB;QACzB,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,UAAU,EAAE,KAAK;QACjB,MAAM,EACJ,iBAAiB,IAAI,gBAAgB,IAAI,kBAAkB;YACzD,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,UAAU;QAChB,eAAe,EAAE,IAAI,CAAC,eAAe;KACtC,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAC/B,QAA4B;IAE5B,OAAO,MAAM,CAAC,WAAW,CACvB,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CACpD,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAU;IAClC,IAAI,CAAC,IAAI,CAAC,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAE7E,MAAM,SAAS,GACb,IAAA,mBAAO,EAAC,IAAI,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;IAElE,OAAO,IAAI,yBAAY,CAAC;QACtB,SAAS;QACT,EAAE,EAAE,OAAO;QACX,WAAW,EAAE,IAAI,CAAC,aAAa;KAChC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,qBAAqB,CAC5B,OAAiD;IAEjD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC5B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAClC,OAAO,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC/B,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClC,OAAO,iDAAiD,GAAG,MAAM,CAAC;IACpE,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAqB,EAAE,WAAmB;IAClE,IAAI,OAAO,YAAY,yBAAY,EAAE,CAAC;QACpC,OAAO,IAAA,2BAAgB,EAAC,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,IAAA,0BAAe,EAAC,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,gBAAgB,CACvB,OAAiD;IAEjD,OAAO,OAAO;SACX,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC;SACjD,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,cAAc,CACrB,OAAgB;IAEhB,OAAO,OAAO,YAAY,wBAAW,IAAI,OAAO,YAAY,yBAAY,CAAC;AAC3E,CAAC;AAED,SAAS,kBAAkB,CACzB,OAAqB,EACrB,KAAa,EACb,WAAmB;IAEnB,MAAM,WAAW,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;IAEhE,IAAI,OAAO,YAAY,wBAAW,EAAE,CAAC;QACnC,OAAO;YACL,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE;YACxC,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE;YAC1C,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;SACxC,CAAC;IACJ,CAAC;IAED,OAAO;QACL,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE;QACxC,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;KACxC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAsB,EACrE,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAyB;IAEtD,IAAI,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,OAAO,gBAAgB,CACrB,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,EAC3C;YACE,OAAO;YACP,gBAAgB,EAAE,IAAI,CAAC,eAAe;YACtC,KAAK;YACL,WAAW;SACZ,CACF,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,YAAY,wBAAW,EAAE,CAAC;QACnC,OAAO,EAAE,QAAQ,EAAE,IAAA,4BAAiB,EAAC,OAAO,CAAC,EAAE,CAAC;IAClD,CAAC;IAED,MAAM,CAAC,IAAI,CACT,QAAQ,IAAI,CAAC,EAAE,+BAA+B,OAAO,CAAC,IAAI,aAAa,CACxE,CAAC;IACF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC3B,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,EACE,OAAO,EACP,MAAM,EACN,WAAW,EACX,SAAS,GAMV,EACD,EACE,OAAO,EACP,gBAAgB,EAChB,KAAK,EACL,WAAW,GAMZ;IAED,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CACnC,kBAAkB,CAAC,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,EAC/C,OAAO,CAAC,SAAS,CAClB,CAAC;IACF,MAAM,cAAc,GAAmB,MAAM,IAAA,2BAAa,EACxD,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,EAC9B;QACE,WAAW;QACX,gBAAgB;QAChB,YAAY,EAAE,OAAO;QACrB,UAAU,EAAE,KAAK;KAClB,CACF;SACE,IAAI,CACH,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE,CAClC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,CAAU,CAC5D;SACA,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;QACtB,MAAM,CAAC,KAAK,CACV,EAAE,GAAG,EAAE,EACP,oCAAoC,OAAO,CAAC,IAAI,UAAU,OAAO,CAAC,EAAE,aAAa,CAClF,CAAC;QAEF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAW,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEL,IAAI,cAAc,CAAC,MAAM,EAAE,CAAC;QAC1B,OAAO;YACL,MAAM,EAAE;gBACN;oBACE,OAAO,CAAC,EAAE;oBACV;wBACE,IAAI,EAAE,WAAW;wBACjB,MAAM,EAAE,QAAQ;wBAChB,GAAG,EAAE,OAAO,CAAC,GAAG;qBACjB;iBACF;aACF;YACD,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,iDAAiD,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM;SACnE,CAAC;IAC/B,CAAC;IAED,OAAO;QACL,MAAM,EAAE;YACN;gBACE,OAAO,CAAC,EAAE;gBACV;oBACE,KAAK,EAAE,cAAc,CAAC,KAAK;oBAC3B,YAAY,EAAE,KAAK;oBACnB,YAAY,EAAE,cAAc,CAAC,YAAY;oBACzC,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,cAAc,CAAC,MAAM;oBAC7B,MAAM,EAAE,UAAU;oBAClB,GAAG,EAAE,OAAO,CAAC,GAAG;iBACjB;aACF;SACF;QACD,QAAQ,EAAE,gBAAgB,CAAC,OAAO,EAAE,WAAW,CAAC;KACtB,CAAC;AAC/B,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Logger } from "pino";
|
|
2
|
+
import type { PostSummary } from "../../client/types.js";
|
|
3
|
+
import type { PostManifestData } from "../manifest/creator.js";
|
|
4
|
+
interface VerifyPostCheckDeps {
|
|
5
|
+
logger: Logger;
|
|
6
|
+
}
|
|
7
|
+
export declare function verifyCompletePost({ logger }: VerifyPostCheckDeps, manifest: PostManifestData, postSummary: PostSummary): Promise<PostManifestData>;
|
|
8
|
+
export {};
|
|
9
|
+
//# sourceMappingURL=verify.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"verify.d.ts","sourceRoot":"","sources":["../../../../src/downloader/post/verify.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAOzD,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,wBAAwB,CAAC;AAEhC,UAAU,mBAAmB;IAC3B,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,kBAAkB,CACtC,EAAE,MAAM,EAAE,EAAE,mBAAmB,EAC/B,QAAQ,EAAE,gBAAgB,EAC1B,WAAW,EAAE,WAAW,GACvB,OAAO,CAAC,gBAAgB,CAAC,CAqB3B"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.verifyCompletePost = verifyCompletePost;
|
|
4
|
+
const promises_1 = require("node:fs/promises");
|
|
5
|
+
const filesystem_js_1 = require("../fs/filesystem.js");
|
|
6
|
+
async function verifyCompletePost({ logger }, manifest, postSummary) {
|
|
7
|
+
const assets = Object.fromEntries(await Promise.all(Object.entries(manifest.assets).map(async ([id, asset]) => [
|
|
8
|
+
id,
|
|
9
|
+
asset ? await verifyAsset({ logger }, asset, postSummary) : asset,
|
|
10
|
+
])));
|
|
11
|
+
const hasFailedAsset = Object.values(assets).some((asset) => asset && asset.status !== "complete");
|
|
12
|
+
return {
|
|
13
|
+
...manifest,
|
|
14
|
+
assets,
|
|
15
|
+
status: hasFailedAsset ? "partial" : "complete",
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
function expectedModifiedTime(asset, postSummary) {
|
|
19
|
+
if (asset.expectedTime) {
|
|
20
|
+
const modifiedTime = (0, filesystem_js_1.normalizeFileTimestamp)(new Date(asset.expectedTime));
|
|
21
|
+
if (!Number.isNaN(modifiedTime.getTime()))
|
|
22
|
+
return modifiedTime;
|
|
23
|
+
}
|
|
24
|
+
return (0, filesystem_js_1.normalizeFileTimestamp)(new Date(new Date(postSummary.updatedDatetime).getTime() +
|
|
25
|
+
asset.contentIndex * 1000));
|
|
26
|
+
}
|
|
27
|
+
function failedAsset(asset, error) {
|
|
28
|
+
return {
|
|
29
|
+
...asset,
|
|
30
|
+
error,
|
|
31
|
+
status: "failed",
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
function obsoleteAsset(asset, error) {
|
|
35
|
+
return {
|
|
36
|
+
...asset,
|
|
37
|
+
error,
|
|
38
|
+
status: "obsolete",
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
async function verifyAsset({ logger }, asset, postSummary) {
|
|
42
|
+
if (asset.status !== "complete") {
|
|
43
|
+
return asset;
|
|
44
|
+
}
|
|
45
|
+
if (asset.contentIndex === undefined) {
|
|
46
|
+
return obsoleteAsset(asset, "content index missing");
|
|
47
|
+
}
|
|
48
|
+
if (!(await (0, filesystem_js_1.exists)(asset.path))) {
|
|
49
|
+
return failedAsset(asset, "file missing");
|
|
50
|
+
}
|
|
51
|
+
const actualSha256 = await (0, filesystem_js_1.hashFile)(asset.path);
|
|
52
|
+
if (actualSha256 !== asset.sha256) {
|
|
53
|
+
return failedAsset(asset, "sha256 mismatch");
|
|
54
|
+
}
|
|
55
|
+
const expectedTime = expectedModifiedTime(asset, postSummary);
|
|
56
|
+
const { mtimeMs } = await (0, promises_1.stat)(asset.path);
|
|
57
|
+
if ((0, filesystem_js_1.normalizeFileTimestamp)(new Date(mtimeMs)).getTime() !==
|
|
58
|
+
expectedTime.getTime()) {
|
|
59
|
+
logger.debug(`Repairing modified time of verified asset ${asset.path} from ${new Date(mtimeMs).toISOString()} to ${expectedTime.toISOString()}`);
|
|
60
|
+
await (0, promises_1.utimes)(asset.path, expectedTime, expectedTime);
|
|
61
|
+
}
|
|
62
|
+
return { ...asset, expectedTime: (0, filesystem_js_1.formatFileTimestamp)(expectedTime) };
|
|
63
|
+
}
|
|
64
|
+
//# sourceMappingURL=verify.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"verify.js","sourceRoot":"","sources":["../../../../src/downloader/post/verify.ts"],"names":[],"mappings":";;AAoBA,gDAyBC;AA7CD,+CAAgD;AAKhD,uDAK6B;AAUtB,KAAK,UAAU,kBAAkB,CACtC,EAAE,MAAM,EAAuB,EAC/B,QAA0B,EAC1B,WAAwB;IAExB,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAC/B,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,CACjC,KAAK,EAAE,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CACpB;QACE,EAAE;QACF,KAAK,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK;KAChB,CACtD,CACF,CACF,CAAC;IACF,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAC/C,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU,CAChD,CAAC;IAEF,OAAO;QACL,GAAG,QAAQ;QACX,MAAM;QACN,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU;KAChD,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,KAAwB,EACxB,WAAwB;IAExB,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;QACvB,MAAM,YAAY,GAAG,IAAA,sCAAsB,EAAC,IAAI,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC;QAC1E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;YAAE,OAAO,YAAY,CAAC;IACjE,CAAC;IAED,OAAO,IAAA,sCAAsB,EAC3B,IAAI,IAAI,CACN,IAAI,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE;QAC7C,KAAK,CAAC,YAAa,GAAG,IAAK,CAC9B,CACF,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAClB,KAAwB,EACxB,KAAa;IAEb,OAAO;QACL,GAAG,KAAK;QACR,KAAK;QACL,MAAM,EAAE,QAAQ;KACjB,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,KAAwB,EACxB,KAAa;IAEb,OAAO;QACL,GAAG,KAAK;QACR,KAAK;QACL,MAAM,EAAE,UAAU;KACnB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,EAAE,MAAM,EAAuB,EAC/B,KAAwB,EACxB,WAAwB;IAExB,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACrC,OAAO,aAAa,CAAC,KAAK,EAAE,uBAAuB,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,CAAC,CAAC,MAAM,IAAA,sBAAM,EAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAChC,OAAO,WAAW,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,IAAA,wBAAQ,EAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChD,IAAI,YAAY,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QAClC,OAAO,WAAW,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAC9D,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAA,eAAI,EAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,IACE,IAAA,sCAAsB,EAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE;QACnD,YAAY,CAAC,OAAO,EAAE,EACtB,CAAC;QACD,MAAM,CAAC,KAAK,CACV,6CAA6C,KAAK,CAAC,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,OAAO,YAAY,CAAC,WAAW,EAAE,EAAE,CACnI,CAAC;QACF,MAAM,IAAA,iBAAM,EAAC,KAAK,CAAC,IAAI,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC;IACvD,CAAC;IAED,OAAO,EAAE,GAAG,KAAK,EAAE,YAAY,EAAE,IAAA,mCAAmB,EAAC,YAAY,CAAC,EAAE,CAAC;AACvE,CAAC"}
|
|
@@ -10,9 +10,11 @@ interface DownloadAssetOptions {
|
|
|
10
10
|
destination: string;
|
|
11
11
|
fallbackDateTime: string;
|
|
12
12
|
mediaContent: MediaContent;
|
|
13
|
+
timeOffset: number;
|
|
13
14
|
}
|
|
14
|
-
export declare function downloadAsset({ headers, logger, transport }: DownloadAssetDeps, { destination, fallbackDateTime, mediaContent }: DownloadAssetOptions): Promise<{
|
|
15
|
+
export declare function downloadAsset({ headers, logger, transport }: DownloadAssetDeps, { destination, fallbackDateTime, mediaContent, timeOffset, }: DownloadAssetOptions): Promise<{
|
|
15
16
|
bytes: number;
|
|
17
|
+
modifiedTime: string;
|
|
16
18
|
sha256: string;
|
|
17
19
|
}>;
|
|
18
20
|
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../../src/downloader/asset/download.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../../src/downloader/asset/download.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAO9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,UAAU,iBAAiB;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,aAAa,CAAC;CAC1B;AAED,UAAU,oBAAoB;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,YAAY,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;CACpB;AAkBD,wBAAsB,aAAa,CACjC,EAAE,OAAY,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,iBAAiB,EACtD,EACE,WAAW,EACX,gBAAgB,EAChB,YAAY,EACZ,UAAU,GACX,EAAE,oBAAoB;;;;GAwDxB"}
|
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { mkdir, rename, utimes } from "node:fs/promises";
|
|
1
|
+
import { createWriteStream } from "node:fs";
|
|
2
|
+
import { mkdir, rename, stat, utimes } from "node:fs/promises";
|
|
4
3
|
import path from "node:path";
|
|
5
4
|
import { pipeline } from "node:stream/promises";
|
|
6
|
-
import { exists, filesize } from "../fs/filesystem.js";
|
|
5
|
+
import { exists, filesize, formatFileTimestamp, hashFile, } from "../fs/filesystem.js";
|
|
7
6
|
class AssetDownloadError extends Error {
|
|
8
7
|
constructor(response, url, body) {
|
|
9
8
|
super(`Asset download failed: ${response.status} ${url}`);
|
|
@@ -14,14 +13,14 @@ class AssetDownloadError extends Error {
|
|
|
14
13
|
this.url = url;
|
|
15
14
|
}
|
|
16
15
|
}
|
|
17
|
-
export async function downloadAsset({ headers = {}, logger, transport }, { destination, fallbackDateTime, mediaContent }) {
|
|
16
|
+
export async function downloadAsset({ headers = {}, logger, transport }, { destination, fallbackDateTime, mediaContent, timeOffset, }) {
|
|
18
17
|
if (await exists(destination)) {
|
|
19
18
|
logger.debug(`Asset ${mediaContent.id} already exists at ${destination}`);
|
|
20
|
-
const [{ size: bytes }, sha256] = await Promise.all([
|
|
21
|
-
|
|
19
|
+
const [{ mtime, size: bytes }, sha256] = await Promise.all([
|
|
20
|
+
stat(destination),
|
|
22
21
|
hashFile(destination),
|
|
23
22
|
]);
|
|
24
|
-
return { bytes, sha256 };
|
|
23
|
+
return { bytes, modifiedTime: formatFileTimestamp(mtime), sha256 };
|
|
25
24
|
}
|
|
26
25
|
logger.debug(`Downloading ${mediaContent.type} asset ${mediaContent.id} to ${destination}`);
|
|
27
26
|
await mkdir(path.dirname(destination), { recursive: true });
|
|
@@ -38,20 +37,18 @@ export async function downloadAsset({ headers = {}, logger, transport }, { desti
|
|
|
38
37
|
}
|
|
39
38
|
await write(tempFilePath, response, downloadedBytes);
|
|
40
39
|
const modified = response.headers.get("Last-Modified");
|
|
41
|
-
const timestamp = modified ? new Date(modified) : new Date(fallbackDateTime);
|
|
40
|
+
const timestamp = offsetTimestamp(modified ? new Date(modified) : new Date(fallbackDateTime), timeOffset);
|
|
42
41
|
await utimes(tempFilePath, timestamp, timestamp);
|
|
43
42
|
const [{ size: bytes }, sha256] = await Promise.all([
|
|
44
43
|
filesize(tempFilePath),
|
|
45
44
|
hashFile(tempFilePath),
|
|
46
45
|
]);
|
|
47
46
|
await rename(tempFilePath, destination);
|
|
48
|
-
|
|
47
|
+
const { mtime } = await stat(destination);
|
|
48
|
+
return { bytes, modifiedTime: formatFileTimestamp(mtime), sha256 };
|
|
49
49
|
}
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
for await (const chunk of createReadStream(filePath))
|
|
53
|
-
hash.update(chunk);
|
|
54
|
-
return hash.digest("hex");
|
|
50
|
+
function offsetTimestamp(timestamp, timeOffset) {
|
|
51
|
+
return new Date(timestamp.getTime() + timeOffset * 1000);
|
|
55
52
|
}
|
|
56
53
|
async function write(path, response, partialBytes) {
|
|
57
54
|
return pipeline(response.body ?? "", createWriteStream(path, {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../../src/downloader/asset/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../../src/downloader/asset/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAKhD,OAAO,EACL,MAAM,EACN,QAAQ,EACR,mBAAmB,EACnB,QAAQ,GACT,MAAM,qBAAqB,CAAC;AAgB7B,MAAM,kBAAmB,SAAQ,KAAK;IAMpC,YAAmB,QAAkB,EAAE,GAAW,EAAE,IAAa;QAC/D,KAAK,CAAC,0BAA0B,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;QACtC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,EAAE,OAAO,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,EAAqB,EACtD,EACE,WAAW,EACX,gBAAgB,EAChB,YAAY,EACZ,UAAU,GACW;IAEvB,IAAI,MAAM,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,EAAE,sBAAsB,WAAW,EAAE,CAAC,CAAC;QAC1E,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACzD,IAAI,CAAC,WAAW,CAAC;YACjB,QAAQ,CAAC,WAAW,CAAC;SACtB,CAAC,CAAC;QAEH,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,mBAAmB,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACrE,CAAC;IAED,MAAM,CAAC,KAAK,CACV,eAAe,YAAY,CAAC,IAAI,UAAU,YAAY,CAAC,EAAE,OAAO,WAAW,EAAE,CAC9E,CAAC;IACF,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,YAAY,GAAG,WAAW,GAAG,OAAO,CAAC;IAE3C,MAAM,cAAc,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IACtC,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC/D,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,KAAK,CACV,wBAAwB,YAAY,CAAC,IAAI,UAAU,YAAY,CAAC,EAAE,OAAO,YAAY,cAAc,eAAe,EAAE,CACrH,CAAC;QACF,cAAc,CAAC,KAAK,GAAG,SAAS,eAAe,GAAG,CAAC;IACrD,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,KAAK,CACpC,IAAI,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAC3D,CAAC;IACF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,kBAAkB,CAC1B,QAAQ,EACR,YAAY,CAAC,GAAG,EAChB,MAAM,QAAQ,CAAC,IAAI,EAAE,CACtB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,CAAC,YAAY,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;IAErD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,eAAe,CAC/B,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC,EAC1D,UAAU,CACX,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAEjD,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAClD,QAAQ,CAAC,YAAY,CAAC;QACtB,QAAQ,CAAC,YAAY,CAAC;KACvB,CAAC,CAAC;IAEH,MAAM,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IACxC,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC;IAE1C,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,mBAAmB,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;AACrE,CAAC;AAED,SAAS,eAAe,CAAC,SAAe,EAAE,UAAkB;IAC1D,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,UAAU,GAAG,IAAK,CAAC,CAAC;AAC5D,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,IAAY,EAAE,QAAkB,EAAE,YAAoB;IACzE,OAAO,QAAQ,CACb,QAAQ,CAAC,IAAI,IAAI,EAAE,EACnB,iBAAiB,CAAC,IAAI,EAAE;QACtB,KAAK,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;KAC/D,CAAC,CACH,CAAC;AACJ,CAAC"}
|
|
@@ -1,19 +1,22 @@
|
|
|
1
1
|
import type { Logger } from "pino";
|
|
2
|
-
export declare const DOWNLOAD_HELP = "Usage: fanbox-dl download [options]\n\nDownload FANBOX posts for selected creators.\n\nSelectors:\n --creator <id> Add a creator ID. Can be repeated.\n --following Add all followed creators.\n --supporting Add all supporting creators.\n --ignore-creator <id> Exclude a creator ID. Can be repeated.\n\nAuth:\n --cookie <value> Raw session ID or FANBOXSESSID=... cookie.\n --cookie-file <path> Read raw cookie or Netscape cookies.txt.\n --user-agent <value> Send the User-Agent from your logged-in browser.\n FANBOX_SESSION_ID Environment fallback.\n
|
|
2
|
+
export declare const DOWNLOAD_HELP = "Usage: fanbox-dl download [options]\n\nDownload FANBOX posts for selected creators.\n\nSelectors:\n --creator <id> Add a creator ID. Can be repeated.\n --following Add all followed creators.\n --supporting Add all supporting creators.\n --ignore-creator <id> Exclude a creator ID. Can be repeated.\n\nAuth:\n --cookie <value> Raw session ID or FANBOXSESSID=... cookie.\n --cookie-file <path> Read raw cookie or Netscape cookies.txt.\n --user-agent <value> Send the User-Agent from your logged-in browser.\n FANBOX_SESSION_ID Environment fallback.\n\nDownload:\n --output <path> Output directory. Default: fanbox-downloads.\n --flat-posts Store post files directly under each creator.\n --max-filename-bytes <n> Max filename bytes including .part temp suffix. Default: 256.\n --flat-parent-min-bytes <n>\n Min optional parent/post name bytes to preserve in flat-posts filenames. Default: 35.\n --verify Verify skipped complete posts against local files.\n\nRequests:\n --concurrency <n> Concurrent requests. Default: 10.\n --http2-session-multiplier <n>\n HTTP/2 sessions per origin multiplier based on concurrency. Default: 10.\n --request-interval-ms <n> Delay between request starts. Default: 500.\n --rate-limit-pause-ms <n> Force overwrite pause ms when 429.\n --max-retries <n> Retry attempts. Default: 3.\n\nOutput:\n --log-level <level> fatal|error|warn|info|debug|trace|silent. Default: info.\n --help Show this help.\n";
|
|
3
3
|
export interface DownloadOptions {
|
|
4
4
|
concurrency: number;
|
|
5
5
|
cookie?: string;
|
|
6
6
|
creatorIds: string[];
|
|
7
|
+
flatParentMinBytes: number;
|
|
7
8
|
flatPosts: boolean;
|
|
8
9
|
following: boolean;
|
|
9
10
|
http2SessionMultiplier: number;
|
|
10
11
|
ignoreCreatorIds: string[];
|
|
12
|
+
maxFilenameBytes: number;
|
|
11
13
|
maxRetries: number;
|
|
12
14
|
output: string;
|
|
13
15
|
rateLimitPauseMs?: number;
|
|
14
16
|
requestIntervalMs: number;
|
|
15
17
|
supporting: boolean;
|
|
16
18
|
userAgent?: string;
|
|
19
|
+
verify: boolean;
|
|
17
20
|
}
|
|
18
21
|
interface ParseDownloadOptionsDeps {
|
|
19
22
|
logger: Logger;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../../../src/downloader/cli/options.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAKnC,eAAO,MAAM,aAAa,
|
|
1
|
+
{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../../../src/downloader/cli/options.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAKnC,eAAO,MAAM,aAAa,ynDAmCzB,CAAC;AAEF,MAAM,WAAW,eAAe;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,OAAO,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,UAAU,wBAAwB;IAChC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,oBAAoB,CAClC,EAAE,MAAM,EAAE,EAAE,wBAAwB,EACpC,IAAI,EAAE,MAAM,EAAE,GACb,eAAe,CA4CjB"}
|
|
@@ -17,11 +17,14 @@ Auth:
|
|
|
17
17
|
--cookie-file <path> Read raw cookie or Netscape cookies.txt.
|
|
18
18
|
--user-agent <value> Send the User-Agent from your logged-in browser.
|
|
19
19
|
FANBOX_SESSION_ID Environment fallback.
|
|
20
|
-
FANBOX_USER_AGENT User-Agent environment fallback.
|
|
21
20
|
|
|
22
21
|
Download:
|
|
23
22
|
--output <path> Output directory. Default: fanbox-downloads.
|
|
24
23
|
--flat-posts Store post files directly under each creator.
|
|
24
|
+
--max-filename-bytes <n> Max filename bytes including .part temp suffix. Default: 256.
|
|
25
|
+
--flat-parent-min-bytes <n>
|
|
26
|
+
Min optional parent/post name bytes to preserve in flat-posts filenames. Default: 35.
|
|
27
|
+
--verify Verify skipped complete posts against local files.
|
|
25
28
|
|
|
26
29
|
Requests:
|
|
27
30
|
--concurrency <n> Concurrent requests. Default: 10.
|
|
@@ -46,10 +49,12 @@ export function parseDownloadOptions({ logger }, args) {
|
|
|
46
49
|
concurrency: parsePositiveInteger("concurrency", values.concurrency),
|
|
47
50
|
cookie,
|
|
48
51
|
creatorIds,
|
|
52
|
+
flatParentMinBytes: parsePositiveInteger("flat-parent-min-bytes", values["flat-parent-min-bytes"]),
|
|
49
53
|
flatPosts: values["flat-posts"],
|
|
50
54
|
following: values.following,
|
|
51
55
|
http2SessionMultiplier: parsePositiveInteger("http2-session-multiplier", values["http2-session-multiplier"]),
|
|
52
56
|
ignoreCreatorIds: values["ignore-creator"] ?? [],
|
|
57
|
+
maxFilenameBytes: parsePositiveInteger("max-filename-bytes", values["max-filename-bytes"]),
|
|
53
58
|
maxRetries: parseNonNegativeInteger("max-retries", values["max-retries"]),
|
|
54
59
|
output: values.output,
|
|
55
60
|
rateLimitPauseMs: values["rate-limit-pause-ms"]
|
|
@@ -58,6 +63,7 @@ export function parseDownloadOptions({ logger }, args) {
|
|
|
58
63
|
requestIntervalMs: parseNonNegativeInteger("request-interval-ms", values["request-interval-ms"]),
|
|
59
64
|
supporting: values.supporting,
|
|
60
65
|
userAgent: values["user-agent"],
|
|
66
|
+
verify: values.verify,
|
|
61
67
|
};
|
|
62
68
|
}
|
|
63
69
|
function parseCookie({ logger }, values) {
|
|
@@ -88,17 +94,20 @@ function parseDownloadArgs(args) {
|
|
|
88
94
|
"cookie-file": { type: "string" },
|
|
89
95
|
creator: { multiple: true, type: "string" },
|
|
90
96
|
"dry-run": { default: false, type: "boolean" },
|
|
97
|
+
"flat-parent-min-bytes": { default: "35", type: "string" },
|
|
91
98
|
"flat-posts": { default: false, type: "boolean" },
|
|
92
99
|
following: { default: false, type: "boolean" },
|
|
93
100
|
"http2-session-multiplier": { default: "10", type: "string" },
|
|
94
101
|
"ignore-creator": { multiple: true, type: "string" },
|
|
95
102
|
"log-level": { default: "info", type: "string" },
|
|
103
|
+
"max-filename-bytes": { default: "256", type: "string" },
|
|
96
104
|
"max-retries": { default: "3", type: "string" },
|
|
97
105
|
output: { default: "fanbox-downloads", type: "string" },
|
|
98
106
|
"rate-limit-pause-ms": { type: "string" },
|
|
99
107
|
"request-interval-ms": { default: "500", type: "string" },
|
|
100
108
|
supporting: { default: false, type: "boolean" },
|
|
101
109
|
"user-agent": { type: "string" },
|
|
110
|
+
verify: { default: false, type: "boolean" },
|
|
102
111
|
},
|
|
103
112
|
strict: true,
|
|
104
113
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"options.js","sourceRoot":"","sources":["../../../../src/downloader/cli/options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAItC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C,MAAM,CAAC,MAAM,aAAa,GAAG
|
|
1
|
+
{"version":3,"file":"options.js","sourceRoot":"","sources":["../../../../src/downloader/cli/options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAItC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C,MAAM,CAAC,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmC5B,CAAC;AAyBF,MAAM,UAAU,oBAAoB,CAClC,EAAE,MAAM,EAA4B,EACpC,IAAc;IAEd,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,EAAE,0BAA0B,CAAC,CAAC;IACnD,MAAM,EAAE,MAAM,EAAE,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IACxC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,UAAU;QACpE,MAAM,IAAI,aAAa,CAAC,2CAA2C,CAAC,CAAC;IAEvE,MAAM,MAAM,GAAG,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,CAAC;IAE/C,OAAO;QACL,WAAW,EAAE,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,WAAW,CAAC;QACpE,MAAM;QACN,UAAU;QACV,kBAAkB,EAAE,oBAAoB,CACtC,uBAAuB,EACvB,MAAM,CAAC,uBAAuB,CAAC,CAChC;QACD,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC;QAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,sBAAsB,EAAE,oBAAoB,CAC1C,0BAA0B,EAC1B,MAAM,CAAC,0BAA0B,CAAC,CACnC;QACD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;QAChD,gBAAgB,EAAE,oBAAoB,CACpC,oBAAoB,EACpB,MAAM,CAAC,oBAAoB,CAAC,CAC7B;QACD,UAAU,EAAE,uBAAuB,CAAC,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QACzE,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,gBAAgB,EAAE,MAAM,CAAC,qBAAqB,CAAC;YAC7C,CAAC,CAAC,uBAAuB,CACrB,qBAAqB,EACrB,MAAM,CAAC,qBAAqB,CAAC,CAC9B;YACH,CAAC,CAAC,SAAS;QACb,iBAAiB,EAAE,uBAAuB,CACxC,qBAAqB,EACrB,MAAM,CAAC,qBAAqB,CAAC,CAC9B;QACD,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;KACtB,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAClB,EAAE,MAAM,EAA4B,EACpC,MAAsD;IAEtD,MAAM,UAAU,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,EAAE,8BAA8B,CAAC,CAAC;QACxE,OAAO,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,EAAE,wBAAwB,CAAC,CAAC;QACvD,OAAO,eAAe,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;QAClC,MAAM,CAAC,KAAK,CACV,EAAE,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,EACpD,+BAA+B,CAChC,CAAC;QACF,OAAO,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACnC,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAc;IACvC,IAAI,CAAC;QACH,OAAO,SAAS,CAAC;YACf,gBAAgB,EAAE,KAAK;YACvB,IAAI;YACJ,OAAO,EAAE;gBACP,WAAW,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC9C,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC1B,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACjC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC3C,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC9C,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC1D,YAAY,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBACjD,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC9C,0BAA0B,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC7D,gBAAgB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACpD,WAAW,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAChD,oBAAoB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACxD,aAAa,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC/C,MAAM,EAAE,EAAE,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACvD,qBAAqB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzC,qBAAqB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzD,UAAU,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC/C,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAChC,MAAM,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;aAC5C;YACD,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,aAAa,CAAE,KAAe,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,IAAY,EAAE,KAAa;IAC1D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,aAAa,CAAC,GAAG,IAAI,iCAAiC,CAAC,CAAC;IACpE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY,EAAE,KAAa;IACvD,MAAM,MAAM,GAAG,uBAAuB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACpD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,MAAM,IAAI,aAAa,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"discover-posts.d.ts","sourceRoot":"","sources":["../../../../src/downloader/creator/discover-posts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAEV,WAAW,EACZ,MAAM,uBAAuB,CAAC;AAE/B,UAAU,gCAAgC;IACxC,MAAM,EAAE,YAAY,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,2BAA2B;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,gCAAgC,EACpD,EACE,SAAS,EACT,OAAO,EACP,sBAAsB,EACtB,KAAW,GACZ,EAAE,2BAA2B,GAC7B,OAAO,CAAC,WAAW,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"discover-posts.d.ts","sourceRoot":"","sources":["../../../../src/downloader/creator/discover-posts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAEV,WAAW,EACZ,MAAM,uBAAuB,CAAC;AAE/B,UAAU,gCAAgC;IACxC,MAAM,EAAE,YAAY,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,2BAA2B;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,gCAAgC,EACpD,EACE,SAAS,EACT,OAAO,EACP,sBAAsB,EACtB,KAAW,GACZ,EAAE,2BAA2B,GAC7B,OAAO,CAAC,WAAW,EAAE,CAAC,CA6BxB"}
|
|
@@ -11,11 +11,13 @@ export async function discoverCreatorPosts({ client, logger }, { creatorId, firs
|
|
|
11
11
|
logger.debug(`Discovered ${posts.length} posts for creator ${creatorId}`);
|
|
12
12
|
if (posts.length < limit)
|
|
13
13
|
return posts;
|
|
14
|
-
const
|
|
14
|
+
const lastPost = posts.at(-1);
|
|
15
|
+
if (!lastPost)
|
|
16
|
+
return posts;
|
|
15
17
|
const remains = await discoverCreatorPosts({ client, logger }, {
|
|
16
18
|
creatorId,
|
|
17
|
-
firstId:
|
|
18
|
-
firstPublishedDatetime:
|
|
19
|
+
firstId: lastPost.id,
|
|
20
|
+
firstPublishedDatetime: lastPost.publishedDatetime,
|
|
19
21
|
limit,
|
|
20
22
|
});
|
|
21
23
|
return [...posts, ...remains.slice(1)];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"discover-posts.js","sourceRoot":"","sources":["../../../../src/downloader/creator/discover-posts.ts"],"names":[],"mappings":"AAoBA,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAoC,EACpD,EACE,SAAS,EACT,OAAO,EACP,sBAAsB,EACtB,KAAK,GAAG,GAAG,GACiB;IAE9B,MAAM,CAAC,KAAK,CACV,mBAAmB,KAAK,eAAe,OAAO,IAAI,OAAO,gBAAgB,SAAS,EAAE,CACrF,CAAC;IACF,MAAM,uBAAuB,GAAG;QAC9B,SAAS;QACT,OAAO;QACP,sBAAsB;QACtB,KAAK;QACL,IAAI,EAAE,QAAQ;KACkB,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;IACrE,MAAM,CAAC,KAAK,CAAC,cAAc,KAAK,CAAC,MAAM,sBAAsB,SAAS,EAAE,CAAC,CAAC;IAC1E,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK;QAAE,OAAO,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"discover-posts.js","sourceRoot":"","sources":["../../../../src/downloader/creator/discover-posts.ts"],"names":[],"mappings":"AAoBA,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAoC,EACpD,EACE,SAAS,EACT,OAAO,EACP,sBAAsB,EACtB,KAAK,GAAG,GAAG,GACiB;IAE9B,MAAM,CAAC,KAAK,CACV,mBAAmB,KAAK,eAAe,OAAO,IAAI,OAAO,gBAAgB,SAAS,EAAE,CACrF,CAAC;IACF,MAAM,uBAAuB,GAAG;QAC9B,SAAS;QACT,OAAO;QACP,sBAAsB;QACtB,KAAK;QACL,IAAI,EAAE,QAAQ;KACkB,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;IACrE,MAAM,CAAC,KAAK,CAAC,cAAc,KAAK,CAAC,MAAM,sBAAsB,SAAS,EAAE,CAAC,CAAC;IAC1E,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK;QAAE,OAAO,KAAK,CAAC;IAEvC,MAAM,QAAQ,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAE5B,MAAM,OAAO,GAAG,MAAM,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAE,EAClB;QACE,SAAS;QACT,OAAO,EAAE,QAAQ,CAAC,EAAE;QACpB,sBAAsB,EAAE,QAAQ,CAAC,iBAAiB;QAClD,KAAK;KACN,CACF,CAAC;IAEF,OAAO,CAAC,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC"}
|