@moontaiworks/fanbox-dl 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -37
- package/dist/cjs/downloader/asset/download.d.ts +3 -1
- package/dist/cjs/downloader/asset/download.d.ts.map +1 -1
- package/dist/cjs/downloader/asset/download.js +11 -14
- package/dist/cjs/downloader/asset/download.js.map +1 -1
- package/dist/cjs/downloader/cli/options.d.ts +2 -1
- package/dist/cjs/downloader/cli/options.d.ts.map +1 -1
- package/dist/cjs/downloader/cli/options.js +3 -1
- package/dist/cjs/downloader/cli/options.js.map +1 -1
- package/dist/cjs/downloader/creator/discover-posts.d.ts.map +1 -1
- package/dist/cjs/downloader/creator/discover-posts.js +5 -3
- package/dist/cjs/downloader/creator/discover-posts.js.map +1 -1
- package/dist/cjs/downloader/creator/sync.d.ts +2 -1
- package/dist/cjs/downloader/creator/sync.d.ts.map +1 -1
- package/dist/cjs/downloader/creator/sync.js +25 -26
- package/dist/cjs/downloader/creator/sync.js.map +1 -1
- package/dist/cjs/downloader/fs/filesystem.d.ts +3 -0
- package/dist/cjs/downloader/fs/filesystem.d.ts.map +1 -1
- package/dist/cjs/downloader/fs/filesystem.js +27 -2
- package/dist/cjs/downloader/fs/filesystem.js.map +1 -1
- package/dist/cjs/downloader/index.d.ts +1 -1
- package/dist/cjs/downloader/index.d.ts.map +1 -1
- package/dist/cjs/downloader/index.js +13 -16
- package/dist/cjs/downloader/index.js.map +1 -1
- package/dist/cjs/downloader/manifest/creator-manager.d.ts +3 -0
- package/dist/cjs/downloader/manifest/creator-manager.d.ts.map +1 -1
- package/dist/cjs/downloader/manifest/creator-manager.js +24 -1
- package/dist/cjs/downloader/manifest/creator-manager.js.map +1 -1
- package/dist/cjs/downloader/manifest/creator.d.ts +6 -0
- package/dist/cjs/downloader/manifest/creator.d.ts.map +1 -1
- package/dist/cjs/downloader/manifest/creator.js +11 -1
- package/dist/cjs/downloader/manifest/creator.js.map +1 -1
- package/dist/cjs/downloader/post/article.d.ts.map +1 -1
- package/dist/cjs/downloader/post/article.js +25 -22
- package/dist/cjs/downloader/post/article.js.map +1 -1
- package/dist/cjs/downloader/post/sync.d.ts +3 -2
- package/dist/cjs/downloader/post/sync.d.ts.map +1 -1
- package/dist/cjs/downloader/post/sync.js +123 -85
- package/dist/cjs/downloader/post/sync.js.map +1 -1
- package/dist/cjs/downloader/post/verify.d.ts +9 -0
- package/dist/cjs/downloader/post/verify.d.ts.map +1 -0
- package/dist/cjs/downloader/post/verify.js +64 -0
- package/dist/cjs/downloader/post/verify.js.map +1 -0
- package/dist/esm/downloader/asset/download.d.ts +3 -1
- package/dist/esm/downloader/asset/download.d.ts.map +1 -1
- package/dist/esm/downloader/asset/download.js +12 -15
- package/dist/esm/downloader/asset/download.js.map +1 -1
- package/dist/esm/downloader/cli/options.d.ts +2 -1
- package/dist/esm/downloader/cli/options.d.ts.map +1 -1
- package/dist/esm/downloader/cli/options.js +3 -1
- package/dist/esm/downloader/cli/options.js.map +1 -1
- package/dist/esm/downloader/creator/discover-posts.d.ts.map +1 -1
- package/dist/esm/downloader/creator/discover-posts.js +5 -3
- package/dist/esm/downloader/creator/discover-posts.js.map +1 -1
- package/dist/esm/downloader/creator/sync.d.ts +2 -1
- package/dist/esm/downloader/creator/sync.d.ts.map +1 -1
- package/dist/esm/downloader/creator/sync.js +25 -26
- package/dist/esm/downloader/creator/sync.js.map +1 -1
- package/dist/esm/downloader/fs/filesystem.d.ts +3 -0
- package/dist/esm/downloader/fs/filesystem.d.ts.map +1 -1
- package/dist/esm/downloader/fs/filesystem.js +24 -2
- package/dist/esm/downloader/fs/filesystem.js.map +1 -1
- package/dist/esm/downloader/index.d.ts +1 -1
- package/dist/esm/downloader/index.d.ts.map +1 -1
- package/dist/esm/downloader/index.js +13 -16
- package/dist/esm/downloader/index.js.map +1 -1
- package/dist/esm/downloader/manifest/creator-manager.d.ts +3 -0
- package/dist/esm/downloader/manifest/creator-manager.d.ts.map +1 -1
- package/dist/esm/downloader/manifest/creator-manager.js +24 -1
- package/dist/esm/downloader/manifest/creator-manager.js.map +1 -1
- package/dist/esm/downloader/manifest/creator.d.ts +6 -0
- package/dist/esm/downloader/manifest/creator.d.ts.map +1 -1
- package/dist/esm/downloader/manifest/creator.js +11 -1
- package/dist/esm/downloader/manifest/creator.js.map +1 -1
- package/dist/esm/downloader/post/article.d.ts.map +1 -1
- package/dist/esm/downloader/post/article.js +25 -22
- package/dist/esm/downloader/post/article.js.map +1 -1
- package/dist/esm/downloader/post/sync.d.ts +3 -2
- package/dist/esm/downloader/post/sync.d.ts.map +1 -1
- package/dist/esm/downloader/post/sync.js +123 -85
- package/dist/esm/downloader/post/sync.js.map +1 -1
- package/dist/esm/downloader/post/verify.d.ts +9 -0
- package/dist/esm/downloader/post/verify.d.ts.map +1 -0
- package/dist/esm/downloader/post/verify.js +61 -0
- package/dist/esm/downloader/post/verify.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.verifyCompletePost = verifyCompletePost;
|
|
4
|
+
const promises_1 = require("node:fs/promises");
|
|
5
|
+
const filesystem_js_1 = require("../fs/filesystem.js");
|
|
6
|
+
async function verifyCompletePost({ logger }, manifest, postSummary) {
|
|
7
|
+
const assets = Object.fromEntries(await Promise.all(Object.entries(manifest.assets).map(async ([id, asset]) => [
|
|
8
|
+
id,
|
|
9
|
+
asset ? await verifyAsset({ logger }, asset, postSummary) : asset,
|
|
10
|
+
])));
|
|
11
|
+
const hasFailedAsset = Object.values(assets).some((asset) => asset && asset.status !== "complete");
|
|
12
|
+
return {
|
|
13
|
+
...manifest,
|
|
14
|
+
assets,
|
|
15
|
+
status: hasFailedAsset ? "partial" : "complete",
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
function expectedModifiedTime(asset, postSummary) {
|
|
19
|
+
if (asset.expectedTime) {
|
|
20
|
+
const modifiedTime = (0, filesystem_js_1.normalizeFileTimestamp)(new Date(asset.expectedTime));
|
|
21
|
+
if (!Number.isNaN(modifiedTime.getTime()))
|
|
22
|
+
return modifiedTime;
|
|
23
|
+
}
|
|
24
|
+
return (0, filesystem_js_1.normalizeFileTimestamp)(new Date(new Date(postSummary.updatedDatetime).getTime() +
|
|
25
|
+
asset.contentIndex * 1000));
|
|
26
|
+
}
|
|
27
|
+
function failedAsset(asset, error) {
|
|
28
|
+
return {
|
|
29
|
+
...asset,
|
|
30
|
+
error,
|
|
31
|
+
status: "failed",
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
function obsoleteAsset(asset, error) {
|
|
35
|
+
return {
|
|
36
|
+
...asset,
|
|
37
|
+
error,
|
|
38
|
+
status: "obsolete",
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
async function verifyAsset({ logger }, asset, postSummary) {
|
|
42
|
+
if (asset.status !== "complete") {
|
|
43
|
+
return asset;
|
|
44
|
+
}
|
|
45
|
+
if (asset.contentIndex === undefined) {
|
|
46
|
+
return obsoleteAsset(asset, "content index missing");
|
|
47
|
+
}
|
|
48
|
+
if (!(await (0, filesystem_js_1.exists)(asset.path))) {
|
|
49
|
+
return failedAsset(asset, "file missing");
|
|
50
|
+
}
|
|
51
|
+
const actualSha256 = await (0, filesystem_js_1.hashFile)(asset.path);
|
|
52
|
+
if (actualSha256 !== asset.sha256) {
|
|
53
|
+
return failedAsset(asset, "sha256 mismatch");
|
|
54
|
+
}
|
|
55
|
+
const expectedTime = expectedModifiedTime(asset, postSummary);
|
|
56
|
+
const { mtimeMs } = await (0, promises_1.stat)(asset.path);
|
|
57
|
+
if ((0, filesystem_js_1.normalizeFileTimestamp)(new Date(mtimeMs)).getTime() !==
|
|
58
|
+
expectedTime.getTime()) {
|
|
59
|
+
logger.debug(`Repairing modified time of verified asset ${asset.path} from ${new Date(mtimeMs).toISOString()} to ${expectedTime.toISOString()}`);
|
|
60
|
+
await (0, promises_1.utimes)(asset.path, expectedTime, expectedTime);
|
|
61
|
+
}
|
|
62
|
+
return { ...asset, expectedTime: (0, filesystem_js_1.formatFileTimestamp)(expectedTime) };
|
|
63
|
+
}
|
|
64
|
+
//# sourceMappingURL=verify.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"verify.js","sourceRoot":"","sources":["../../../../src/downloader/post/verify.ts"],"names":[],"mappings":";;AAoBA,gDAyBC;AA7CD,+CAAgD;AAKhD,uDAK6B;AAUtB,KAAK,UAAU,kBAAkB,CACtC,EAAE,MAAM,EAAuB,EAC/B,QAA0B,EAC1B,WAAwB;IAExB,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAC/B,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,CACjC,KAAK,EAAE,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CACpB;QACE,EAAE;QACF,KAAK,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK;KAChB,CACtD,CACF,CACF,CAAC;IACF,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAC/C,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU,CAChD,CAAC;IAEF,OAAO;QACL,GAAG,QAAQ;QACX,MAAM;QACN,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU;KAChD,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,KAAwB,EACxB,WAAwB;IAExB,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;QACvB,MAAM,YAAY,GAAG,IAAA,sCAAsB,EAAC,IAAI,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC;QAC1E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;YAAE,OAAO,YAAY,CAAC;IACjE,CAAC;IAED,OAAO,IAAA,sCAAsB,EAC3B,IAAI,IAAI,CACN,IAAI,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE;QAC7C,KAAK,CAAC,YAAa,GAAG,IAAK,CAC9B,CACF,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAClB,KAAwB,EACxB,KAAa;IAEb,OAAO;QACL,GAAG,KAAK;QACR,KAAK;QACL,MAAM,EAAE,QAAQ;KACjB,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,KAAwB,EACxB,KAAa;IAEb,OAAO;QACL,GAAG,KAAK;QACR,KAAK;QACL,MAAM,EAAE,UAAU;KACnB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,EAAE,MAAM,EAAuB,EAC/B,KAAwB,EACxB,WAAwB;IAExB,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACrC,OAAO,aAAa,CAAC,KAAK,EAAE,uBAAuB,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,CAAC,CAAC,MAAM,IAAA,sBAAM,EAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAChC,OAAO,WAAW,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,IAAA,wBAAQ,EAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChD,IAAI,YAAY,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QAClC,OAAO,WAAW,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAC9D,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAA,eAAI,EAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,IACE,IAAA,sCAAsB,EAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE;QACnD,YAAY,CAAC,OAAO,EAAE,EACtB,CAAC;QACD,MAAM,CAAC,KAAK,CACV,6CAA6C,KAAK,CAAC,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,OAAO,YAAY,CAAC,WAAW,EAAE,EAAE,CACnI,CAAC;QACF,MAAM,IAAA,iBAAM,EAAC,KAAK,CAAC,IAAI,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC;IACvD,CAAC;IAED,OAAO,EAAE,GAAG,KAAK,EAAE,YAAY,EAAE,IAAA,mCAAmB,EAAC,YAAY,CAAC,EAAE,CAAC;AACvE,CAAC"}
|
|
@@ -10,9 +10,11 @@ interface DownloadAssetOptions {
|
|
|
10
10
|
destination: string;
|
|
11
11
|
fallbackDateTime: string;
|
|
12
12
|
mediaContent: MediaContent;
|
|
13
|
+
timeOffset: number;
|
|
13
14
|
}
|
|
14
|
-
export declare function downloadAsset({ headers, logger, transport }: DownloadAssetDeps, { destination, fallbackDateTime, mediaContent }: DownloadAssetOptions): Promise<{
|
|
15
|
+
export declare function downloadAsset({ headers, logger, transport }: DownloadAssetDeps, { destination, fallbackDateTime, mediaContent, timeOffset, }: DownloadAssetOptions): Promise<{
|
|
15
16
|
bytes: number;
|
|
17
|
+
modifiedTime: string;
|
|
16
18
|
sha256: string;
|
|
17
19
|
}>;
|
|
18
20
|
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../../src/downloader/asset/download.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../../src/downloader/asset/download.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAO9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,UAAU,iBAAiB;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,aAAa,CAAC;CAC1B;AAED,UAAU,oBAAoB;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,YAAY,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;CACpB;AAkBD,wBAAsB,aAAa,CACjC,EAAE,OAAY,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,iBAAiB,EACtD,EACE,WAAW,EACX,gBAAgB,EAChB,YAAY,EACZ,UAAU,GACX,EAAE,oBAAoB;;;;GAwDxB"}
|
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { mkdir, rename, utimes } from "node:fs/promises";
|
|
1
|
+
import { createWriteStream } from "node:fs";
|
|
2
|
+
import { mkdir, rename, stat, utimes } from "node:fs/promises";
|
|
4
3
|
import path from "node:path";
|
|
5
4
|
import { pipeline } from "node:stream/promises";
|
|
6
|
-
import { exists, filesize } from "../fs/filesystem.js";
|
|
5
|
+
import { exists, filesize, formatFileTimestamp, hashFile, } from "../fs/filesystem.js";
|
|
7
6
|
class AssetDownloadError extends Error {
|
|
8
7
|
constructor(response, url, body) {
|
|
9
8
|
super(`Asset download failed: ${response.status} ${url}`);
|
|
@@ -14,14 +13,14 @@ class AssetDownloadError extends Error {
|
|
|
14
13
|
this.url = url;
|
|
15
14
|
}
|
|
16
15
|
}
|
|
17
|
-
export async function downloadAsset({ headers = {}, logger, transport }, { destination, fallbackDateTime, mediaContent }) {
|
|
16
|
+
export async function downloadAsset({ headers = {}, logger, transport }, { destination, fallbackDateTime, mediaContent, timeOffset, }) {
|
|
18
17
|
if (await exists(destination)) {
|
|
19
18
|
logger.debug(`Asset ${mediaContent.id} already exists at ${destination}`);
|
|
20
|
-
const [{ size: bytes }, sha256] = await Promise.all([
|
|
21
|
-
|
|
19
|
+
const [{ mtime, size: bytes }, sha256] = await Promise.all([
|
|
20
|
+
stat(destination),
|
|
22
21
|
hashFile(destination),
|
|
23
22
|
]);
|
|
24
|
-
return { bytes, sha256 };
|
|
23
|
+
return { bytes, modifiedTime: formatFileTimestamp(mtime), sha256 };
|
|
25
24
|
}
|
|
26
25
|
logger.debug(`Downloading ${mediaContent.type} asset ${mediaContent.id} to ${destination}`);
|
|
27
26
|
await mkdir(path.dirname(destination), { recursive: true });
|
|
@@ -38,20 +37,18 @@ export async function downloadAsset({ headers = {}, logger, transport }, { desti
|
|
|
38
37
|
}
|
|
39
38
|
await write(tempFilePath, response, downloadedBytes);
|
|
40
39
|
const modified = response.headers.get("Last-Modified");
|
|
41
|
-
const timestamp = modified ? new Date(modified) : new Date(fallbackDateTime);
|
|
40
|
+
const timestamp = offsetTimestamp(modified ? new Date(modified) : new Date(fallbackDateTime), timeOffset);
|
|
42
41
|
await utimes(tempFilePath, timestamp, timestamp);
|
|
43
42
|
const [{ size: bytes }, sha256] = await Promise.all([
|
|
44
43
|
filesize(tempFilePath),
|
|
45
44
|
hashFile(tempFilePath),
|
|
46
45
|
]);
|
|
47
46
|
await rename(tempFilePath, destination);
|
|
48
|
-
|
|
47
|
+
const { mtime } = await stat(destination);
|
|
48
|
+
return { bytes, modifiedTime: formatFileTimestamp(mtime), sha256 };
|
|
49
49
|
}
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
for await (const chunk of createReadStream(filePath))
|
|
53
|
-
hash.update(chunk);
|
|
54
|
-
return hash.digest("hex");
|
|
50
|
+
function offsetTimestamp(timestamp, timeOffset) {
|
|
51
|
+
return new Date(timestamp.getTime() + timeOffset * 1000);
|
|
55
52
|
}
|
|
56
53
|
async function write(path, response, partialBytes) {
|
|
57
54
|
return pipeline(response.body ?? "", createWriteStream(path, {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../../src/downloader/asset/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../../src/downloader/asset/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAKhD,OAAO,EACL,MAAM,EACN,QAAQ,EACR,mBAAmB,EACnB,QAAQ,GACT,MAAM,qBAAqB,CAAC;AAgB7B,MAAM,kBAAmB,SAAQ,KAAK;IAMpC,YAAmB,QAAkB,EAAE,GAAW,EAAE,IAAa;QAC/D,KAAK,CAAC,0BAA0B,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;QACtC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,EAAE,OAAO,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,EAAqB,EACtD,EACE,WAAW,EACX,gBAAgB,EAChB,YAAY,EACZ,UAAU,GACW;IAEvB,IAAI,MAAM,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,EAAE,sBAAsB,WAAW,EAAE,CAAC,CAAC;QAC1E,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACzD,IAAI,CAAC,WAAW,CAAC;YACjB,QAAQ,CAAC,WAAW,CAAC;SACtB,CAAC,CAAC;QAEH,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,mBAAmB,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACrE,CAAC;IAED,MAAM,CAAC,KAAK,CACV,eAAe,YAAY,CAAC,IAAI,UAAU,YAAY,CAAC,EAAE,OAAO,WAAW,EAAE,CAC9E,CAAC;IACF,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,YAAY,GAAG,WAAW,GAAG,OAAO,CAAC;IAE3C,MAAM,cAAc,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IACtC,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC/D,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,KAAK,CACV,wBAAwB,YAAY,CAAC,IAAI,UAAU,YAAY,CAAC,EAAE,OAAO,YAAY,cAAc,eAAe,EAAE,CACrH,CAAC;QACF,cAAc,CAAC,KAAK,GAAG,SAAS,eAAe,GAAG,CAAC;IACrD,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,KAAK,CACpC,IAAI,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAC3D,CAAC;IACF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,kBAAkB,CAC1B,QAAQ,EACR,YAAY,CAAC,GAAG,EAChB,MAAM,QAAQ,CAAC,IAAI,EAAE,CACtB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,CAAC,YAAY,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAC;IAErD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,eAAe,CAC/B,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC,EAC1D,UAAU,CACX,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAEjD,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAClD,QAAQ,CAAC,YAAY,CAAC;QACtB,QAAQ,CAAC,YAAY,CAAC;KACvB,CAAC,CAAC;IAEH,MAAM,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IACxC,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC;IAE1C,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,mBAAmB,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;AACrE,CAAC;AAED,SAAS,eAAe,CAAC,SAAe,EAAE,UAAkB;IAC1D,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,UAAU,GAAG,IAAK,CAAC,CAAC;AAC5D,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,IAAY,EAAE,QAAkB,EAAE,YAAoB;IACzE,OAAO,QAAQ,CACb,QAAQ,CAAC,IAAI,IAAI,EAAE,EACnB,iBAAiB,CAAC,IAAI,EAAE;QACtB,KAAK,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;KAC/D,CAAC,CACH,CAAC;AACJ,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Logger } from "pino";
|
|
2
|
-
export declare const DOWNLOAD_HELP = "Usage: fanbox-dl download [options]\n\nDownload FANBOX posts for selected creators.\n\nSelectors:\n --creator <id> Add a creator ID. Can be repeated.\n --following Add all followed creators.\n --supporting Add all supporting creators.\n --ignore-creator <id> Exclude a creator ID. Can be repeated.\n\nAuth:\n --cookie <value> Raw session ID or FANBOXSESSID=... cookie.\n --cookie-file <path> Read raw cookie or Netscape cookies.txt.\n --user-agent <value> Send the User-Agent from your logged-in browser.\n FANBOX_SESSION_ID Environment fallback.\n
|
|
2
|
+
export declare const DOWNLOAD_HELP = "Usage: fanbox-dl download [options]\n\nDownload FANBOX posts for selected creators.\n\nSelectors:\n --creator <id> Add a creator ID. Can be repeated.\n --following Add all followed creators.\n --supporting Add all supporting creators.\n --ignore-creator <id> Exclude a creator ID. Can be repeated.\n\nAuth:\n --cookie <value> Raw session ID or FANBOXSESSID=... cookie.\n --cookie-file <path> Read raw cookie or Netscape cookies.txt.\n --user-agent <value> Send the User-Agent from your logged-in browser.\n FANBOX_SESSION_ID Environment fallback.\n\nDownload:\n --output <path> Output directory. Default: fanbox-downloads.\n --flat-posts Store post files directly under each creator.\n --max-filename-bytes <n> Max filename bytes including .part temp suffix. Default: 256.\n --flat-parent-min-bytes <n>\n Min optional parent/post name bytes to preserve in flat-posts filenames. Default: 35.\n --verify Verify skipped complete posts against local files.\n\nRequests:\n --concurrency <n> Concurrent requests. Default: 10.\n --http2-session-multiplier <n>\n HTTP/2 sessions per origin multiplier based on concurrency. Default: 10.\n --request-interval-ms <n> Delay between request starts. Default: 500.\n --rate-limit-pause-ms <n> Force overwrite pause ms when 429.\n --max-retries <n> Retry attempts. Default: 3.\n\nOutput:\n --log-level <level> fatal|error|warn|info|debug|trace|silent. Default: info.\n --help Show this help.\n";
|
|
3
3
|
export interface DownloadOptions {
|
|
4
4
|
concurrency: number;
|
|
5
5
|
cookie?: string;
|
|
@@ -16,6 +16,7 @@ export interface DownloadOptions {
|
|
|
16
16
|
requestIntervalMs: number;
|
|
17
17
|
supporting: boolean;
|
|
18
18
|
userAgent?: string;
|
|
19
|
+
verify: boolean;
|
|
19
20
|
}
|
|
20
21
|
interface ParseDownloadOptionsDeps {
|
|
21
22
|
logger: Logger;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../../../src/downloader/cli/options.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAKnC,eAAO,MAAM,aAAa,
|
|
1
|
+
{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../../../src/downloader/cli/options.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAKnC,eAAO,MAAM,aAAa,ynDAmCzB,CAAC;AAEF,MAAM,WAAW,eAAe;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,OAAO,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,UAAU,wBAAwB;IAChC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,oBAAoB,CAClC,EAAE,MAAM,EAAE,EAAE,wBAAwB,EACpC,IAAI,EAAE,MAAM,EAAE,GACb,eAAe,CA4CjB"}
|
|
@@ -17,7 +17,6 @@ Auth:
|
|
|
17
17
|
--cookie-file <path> Read raw cookie or Netscape cookies.txt.
|
|
18
18
|
--user-agent <value> Send the User-Agent from your logged-in browser.
|
|
19
19
|
FANBOX_SESSION_ID Environment fallback.
|
|
20
|
-
FANBOX_USER_AGENT User-Agent environment fallback.
|
|
21
20
|
|
|
22
21
|
Download:
|
|
23
22
|
--output <path> Output directory. Default: fanbox-downloads.
|
|
@@ -25,6 +24,7 @@ Download:
|
|
|
25
24
|
--max-filename-bytes <n> Max filename bytes including .part temp suffix. Default: 256.
|
|
26
25
|
--flat-parent-min-bytes <n>
|
|
27
26
|
Min optional parent/post name bytes to preserve in flat-posts filenames. Default: 35.
|
|
27
|
+
--verify Verify skipped complete posts against local files.
|
|
28
28
|
|
|
29
29
|
Requests:
|
|
30
30
|
--concurrency <n> Concurrent requests. Default: 10.
|
|
@@ -63,6 +63,7 @@ export function parseDownloadOptions({ logger }, args) {
|
|
|
63
63
|
requestIntervalMs: parseNonNegativeInteger("request-interval-ms", values["request-interval-ms"]),
|
|
64
64
|
supporting: values.supporting,
|
|
65
65
|
userAgent: values["user-agent"],
|
|
66
|
+
verify: values.verify,
|
|
66
67
|
};
|
|
67
68
|
}
|
|
68
69
|
function parseCookie({ logger }, values) {
|
|
@@ -106,6 +107,7 @@ function parseDownloadArgs(args) {
|
|
|
106
107
|
"request-interval-ms": { default: "500", type: "string" },
|
|
107
108
|
supporting: { default: false, type: "boolean" },
|
|
108
109
|
"user-agent": { type: "string" },
|
|
110
|
+
verify: { default: false, type: "boolean" },
|
|
109
111
|
},
|
|
110
112
|
strict: true,
|
|
111
113
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"options.js","sourceRoot":"","sources":["../../../../src/downloader/cli/options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAItC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C,MAAM,CAAC,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmC5B,CAAC;
|
|
1
|
+
{"version":3,"file":"options.js","sourceRoot":"","sources":["../../../../src/downloader/cli/options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAItC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C,MAAM,CAAC,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmC5B,CAAC;AAyBF,MAAM,UAAU,oBAAoB,CAClC,EAAE,MAAM,EAA4B,EACpC,IAAc;IAEd,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,EAAE,0BAA0B,CAAC,CAAC;IACnD,MAAM,EAAE,MAAM,EAAE,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IACxC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,UAAU;QACpE,MAAM,IAAI,aAAa,CAAC,2CAA2C,CAAC,CAAC;IAEvE,MAAM,MAAM,GAAG,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,CAAC;IAE/C,OAAO;QACL,WAAW,EAAE,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,WAAW,CAAC;QACpE,MAAM;QACN,UAAU;QACV,kBAAkB,EAAE,oBAAoB,CACtC,uBAAuB,EACvB,MAAM,CAAC,uBAAuB,CAAC,CAChC;QACD,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC;QAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,sBAAsB,EAAE,oBAAoB,CAC1C,0BAA0B,EAC1B,MAAM,CAAC,0BAA0B,CAAC,CACnC;QACD,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;QAChD,gBAAgB,EAAE,oBAAoB,CACpC,oBAAoB,EACpB,MAAM,CAAC,oBAAoB,CAAC,CAC7B;QACD,UAAU,EAAE,uBAAuB,CAAC,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QACzE,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,gBAAgB,EAAE,MAAM,CAAC,qBAAqB,CAAC;YAC7C,CAAC,CAAC,uBAAuB,CACrB,qBAAqB,EACrB,MAAM,CAAC,qBAAqB,CAAC,CAC9B;YACH,CAAC,CAAC,SAAS;QACb,iBAAiB,EAAE,uBAAuB,CACxC,qBAAqB,EACrB,MAAM,CAAC,qBAAqB,CAAC,CAC9B;QACD,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;KACtB,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAClB,EAAE,MAAM,EAA4B,EACpC,MAAsD;IAEtD,MAAM,UAAU,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,EAAE,8BAA8B,CAAC,CAAC;QACxE,OAAO,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,EAAE,wBAAwB,CAAC,CAAC;QACvD,OAAO,eAAe,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;QAClC,MAAM,CAAC,KAAK,CACV,EAAE,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,EACpD,+BAA+B,CAChC,CAAC;QACF,OAAO,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACnC,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAc;IACvC,IAAI,CAAC;QACH,OAAO,SAAS,CAAC;YACf,gBAAgB,EAAE,KAAK;YACvB,IAAI;YACJ,OAAO,EAAE;gBACP,WAAW,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC9C,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC1B,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACjC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC3C,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC9C,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC1D,YAAY,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBACjD,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC9C,0BAA0B,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC7D,gBAAgB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACpD,WAAW,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAChD,oBAAoB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACxD,aAAa,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC/C,MAAM,EAAE,EAAE,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACvD,qBAAqB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzC,qBAAqB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzD,UAAU,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC/C,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAChC,MAAM,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE;aAC5C;YACD,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,aAAa,CAAE,KAAe,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,IAAY,EAAE,KAAa;IAC1D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,aAAa,CAAC,GAAG,IAAI,iCAAiC,CAAC,CAAC;IACpE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY,EAAE,KAAa;IACvD,MAAM,MAAM,GAAG,uBAAuB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACpD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,MAAM,IAAI,aAAa,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"discover-posts.d.ts","sourceRoot":"","sources":["../../../../src/downloader/creator/discover-posts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAEV,WAAW,EACZ,MAAM,uBAAuB,CAAC;AAE/B,UAAU,gCAAgC;IACxC,MAAM,EAAE,YAAY,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,2BAA2B;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,gCAAgC,EACpD,EACE,SAAS,EACT,OAAO,EACP,sBAAsB,EACtB,KAAW,GACZ,EAAE,2BAA2B,GAC7B,OAAO,CAAC,WAAW,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"discover-posts.d.ts","sourceRoot":"","sources":["../../../../src/downloader/creator/discover-posts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAEV,WAAW,EACZ,MAAM,uBAAuB,CAAC;AAE/B,UAAU,gCAAgC;IACxC,MAAM,EAAE,YAAY,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,2BAA2B;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,gCAAgC,EACpD,EACE,SAAS,EACT,OAAO,EACP,sBAAsB,EACtB,KAAW,GACZ,EAAE,2BAA2B,GAC7B,OAAO,CAAC,WAAW,EAAE,CAAC,CA6BxB"}
|
|
@@ -11,11 +11,13 @@ export async function discoverCreatorPosts({ client, logger }, { creatorId, firs
|
|
|
11
11
|
logger.debug(`Discovered ${posts.length} posts for creator ${creatorId}`);
|
|
12
12
|
if (posts.length < limit)
|
|
13
13
|
return posts;
|
|
14
|
-
const
|
|
14
|
+
const lastPost = posts.at(-1);
|
|
15
|
+
if (!lastPost)
|
|
16
|
+
return posts;
|
|
15
17
|
const remains = await discoverCreatorPosts({ client, logger }, {
|
|
16
18
|
creatorId,
|
|
17
|
-
firstId:
|
|
18
|
-
firstPublishedDatetime:
|
|
19
|
+
firstId: lastPost.id,
|
|
20
|
+
firstPublishedDatetime: lastPost.publishedDatetime,
|
|
19
21
|
limit,
|
|
20
22
|
});
|
|
21
23
|
return [...posts, ...remains.slice(1)];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"discover-posts.js","sourceRoot":"","sources":["../../../../src/downloader/creator/discover-posts.ts"],"names":[],"mappings":"AAoBA,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAoC,EACpD,EACE,SAAS,EACT,OAAO,EACP,sBAAsB,EACtB,KAAK,GAAG,GAAG,GACiB;IAE9B,MAAM,CAAC,KAAK,CACV,mBAAmB,KAAK,eAAe,OAAO,IAAI,OAAO,gBAAgB,SAAS,EAAE,CACrF,CAAC;IACF,MAAM,uBAAuB,GAAG;QAC9B,SAAS;QACT,OAAO;QACP,sBAAsB;QACtB,KAAK;QACL,IAAI,EAAE,QAAQ;KACkB,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;IACrE,MAAM,CAAC,KAAK,CAAC,cAAc,KAAK,CAAC,MAAM,sBAAsB,SAAS,EAAE,CAAC,CAAC;IAC1E,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK;QAAE,OAAO,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"discover-posts.js","sourceRoot":"","sources":["../../../../src/downloader/creator/discover-posts.ts"],"names":[],"mappings":"AAoBA,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAoC,EACpD,EACE,SAAS,EACT,OAAO,EACP,sBAAsB,EACtB,KAAK,GAAG,GAAG,GACiB;IAE9B,MAAM,CAAC,KAAK,CACV,mBAAmB,KAAK,eAAe,OAAO,IAAI,OAAO,gBAAgB,SAAS,EAAE,CACrF,CAAC;IACF,MAAM,uBAAuB,GAAG;QAC9B,SAAS;QACT,OAAO;QACP,sBAAsB;QACtB,KAAK;QACL,IAAI,EAAE,QAAQ;KACkB,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;IACrE,MAAM,CAAC,KAAK,CAAC,cAAc,KAAK,CAAC,MAAM,sBAAsB,SAAS,EAAE,CAAC,CAAC;IAC1E,IAAI,KAAK,CAAC,MAAM,GAAG,KAAK;QAAE,OAAO,KAAK,CAAC;IAEvC,MAAM,QAAQ,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAE5B,MAAM,OAAO,GAAG,MAAM,oBAAoB,CACxC,EAAE,MAAM,EAAE,MAAM,EAAE,EAClB;QACE,SAAS;QACT,OAAO,EAAE,QAAQ,CAAC,EAAE;QACpB,sBAAsB,EAAE,QAAQ,CAAC,iBAAiB;QAClD,KAAK;KACN,CACF,CAAC;IAEF,OAAO,CAAC,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC"}
|
|
@@ -10,7 +10,8 @@ interface SyncCreatorDeps {
|
|
|
10
10
|
manifest: CreatorManifest;
|
|
11
11
|
pathManager: PathManager;
|
|
12
12
|
transport: HttpTransport;
|
|
13
|
+
verify?: boolean;
|
|
13
14
|
}
|
|
14
|
-
export declare function syncCreator({ client, headers, logger, manifest, pathManager, transport, }: SyncCreatorDeps): Promise<void>;
|
|
15
|
+
export declare function syncCreator({ client, headers, logger, manifest, pathManager, transport, verify, }: SyncCreatorDeps): Promise<void>;
|
|
15
16
|
export {};
|
|
16
17
|
//# sourceMappingURL=sync.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../../../src/downloader/creator/sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAoB,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../../../src/downloader/creator/sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAoB,MAAM,wBAAwB,CAAC;AAUhF,UAAU,eAAe;IACvB,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,eAAe,CAAC;IAC1B,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,aAAa,CAAC;IACzB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,wBAAsB,WAAW,CAAC,EAChC,MAAM,EACN,OAAO,EACP,MAAM,EACN,QAAQ,EACR,WAAW,EACX,SAAS,EACT,MAAc,GACf,EAAE,eAAe,iBAoEjB"}
|
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { preSyncPostCheck, syncPost } from "../post/sync.js";
|
|
2
2
|
import { discoverCreatorPosts } from "./discover-posts.js";
|
|
3
|
-
export async function syncCreator({ client, headers, logger, manifest, pathManager, transport, }) {
|
|
3
|
+
export async function syncCreator({ client, headers, logger, manifest, pathManager, transport, verify = false, }) {
|
|
4
4
|
const postSummaries = await discoverCreatorPosts({ client, logger }, { creatorId: manifest.creatorId });
|
|
5
5
|
logger.info(`Discovered total ${postSummaries.length} posts for creator ${manifest.creatorId}`);
|
|
6
6
|
const processingPosts = [];
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
logger.debug(`Initializing ${
|
|
10
|
-
const preCheckResult = preSyncPostCheck({ logger, manifest }, postSummary);
|
|
11
|
-
if (preCheckResult.status
|
|
7
|
+
for (const [index, postSummary] of postSummaries.entries()) {
|
|
8
|
+
const postIndex = index + 1;
|
|
9
|
+
logger.debug(`Initializing ${postIndex}/${postSummaries.length} post ${postSummary.id} for creator ${manifest.creatorId}`);
|
|
10
|
+
const preCheckResult = await preSyncPostCheck({ logger, manifest, verify }, postSummary);
|
|
11
|
+
if (preCheckResult.status === "complete" ||
|
|
12
|
+
preCheckResult.status === "skipped") {
|
|
12
13
|
manifest.posts[postSummary.id] = preCheckResult;
|
|
13
14
|
continue;
|
|
14
15
|
}
|
|
@@ -16,16 +17,8 @@ export async function syncCreator({ client, headers, logger, manifest, pathManag
|
|
|
16
17
|
const post = await client
|
|
17
18
|
.getPost({ postId: postSummary.id })
|
|
18
19
|
.catch(async (err) => {
|
|
19
|
-
manifest.
|
|
20
|
-
|
|
21
|
-
error: String(err),
|
|
22
|
-
id: postSummary.id,
|
|
23
|
-
restricted: postSummary.isRestricted,
|
|
24
|
-
status: "failed",
|
|
25
|
-
updatedDatetime: postSummary.updatedDatetime,
|
|
26
|
-
};
|
|
27
|
-
logger.error({ err }, `Error occurred while fetch post manifest of ${index}/${postSummaries.length} post ${postSummary.id}, skipping.`);
|
|
28
|
-
await manifest.save();
|
|
20
|
+
logger.error({ err }, `Error occurred while fetch post manifest of ${postIndex}/${postSummaries.length} post ${postSummary.id}, skipping.`);
|
|
21
|
+
await saveFailedPostManifest(manifest, postSummary, err);
|
|
29
22
|
});
|
|
30
23
|
if (!post)
|
|
31
24
|
continue;
|
|
@@ -35,20 +28,26 @@ export async function syncCreator({ client, headers, logger, manifest, pathManag
|
|
|
35
28
|
return manifest.save();
|
|
36
29
|
})
|
|
37
30
|
.catch((err) => {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
error: String(err),
|
|
41
|
-
id: postSummary.id,
|
|
42
|
-
restricted: postSummary.isRestricted,
|
|
43
|
-
status: "failed",
|
|
44
|
-
updatedDatetime: postSummary.updatedDatetime,
|
|
45
|
-
};
|
|
46
|
-
logger.error({ err }, `Error occurred while syncing ${index}/${postSummaries.length} post ${postSummary.id}, skipping.`);
|
|
47
|
-
return manifest.save();
|
|
31
|
+
logger.error({ err }, `Error occurred while syncing ${postIndex}/${postSummaries.length} post ${postSummary.id}, skipping.`);
|
|
32
|
+
return saveFailedPostManifest(manifest, postSummary, err);
|
|
48
33
|
});
|
|
49
34
|
processingPosts.push(syncPostPromise);
|
|
50
35
|
}
|
|
51
36
|
logger.info(`All ${postSummaries.length} posts of creator ${manifest.creatorId} have been initialized for sync, waiting for all to complete...`);
|
|
52
37
|
await Promise.all(processingPosts);
|
|
53
38
|
}
|
|
39
|
+
function failedPostManifestData(postSummary, err) {
|
|
40
|
+
return {
|
|
41
|
+
assets: {},
|
|
42
|
+
error: String(err),
|
|
43
|
+
id: postSummary.id,
|
|
44
|
+
restricted: postSummary.isRestricted,
|
|
45
|
+
status: "failed",
|
|
46
|
+
updatedDatetime: postSummary.updatedDatetime,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
async function saveFailedPostManifest(manifest, postSummary, err) {
|
|
50
|
+
manifest.posts[postSummary.id] = failedPostManifestData(postSummary, err);
|
|
51
|
+
await manifest.save();
|
|
52
|
+
}
|
|
54
53
|
//# sourceMappingURL=sync.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../../../src/downloader/creator/sync.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../../../src/downloader/creator/sync.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAkB3D,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAChC,MAAM,EACN,OAAO,EACP,MAAM,EACN,QAAQ,EACR,WAAW,EACX,SAAS,EACT,MAAM,GAAG,KAAK,GACE;IAChB,MAAM,aAAa,GAAG,MAAM,oBAAoB,CAC9C,EAAE,MAAM,EAAE,MAAM,EAAE,EAClB,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,CAClC,CAAC;IACF,MAAM,CAAC,IAAI,CACT,oBAAoB,aAAa,CAAC,MAAM,sBAAsB,QAAQ,CAAC,SAAS,EAAE,CACnF,CAAC;IAEF,MAAM,eAAe,GAAoB,EAAE,CAAC;IAE5C,KAAK,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3D,MAAM,SAAS,GAAG,KAAK,GAAG,CAAC,CAAC;QAC5B,MAAM,CAAC,KAAK,CACV,gBAAgB,SAAS,IAAI,aAAa,CAAC,MAAM,SAAS,WAAW,CAAC,EAAE,gBAAgB,QAAQ,CAAC,SAAS,EAAE,CAC7G,CAAC;QACF,MAAM,cAAc,GAAG,MAAM,gBAAgB,CAC3C,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAC5B,WAAW,CACZ,CAAC;QAEF,IACE,cAAc,CAAC,MAAM,KAAK,UAAU;YACpC,cAAc,CAAC,MAAM,KAAK,SAAS,EACnC,CAAC;YACD,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC;YAChD,SAAS;QACX,CAAC;QAKD,MAAM,eAAe,GAAG,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtD,MAAM,IAAI,GAAG,MAAM,MAAM;aACtB,OAAO,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC;aACnC,KAAK,CAAC,KAAK,EAAE,GAAY,EAAE,EAAE;YAC5B,MAAM,CAAC,KAAK,CACV,EAAE,GAAG,EAAE,EACP,+CAA+C,SAAS,IAAI,aAAa,CAAC,MAAM,SAAS,WAAW,CAAC,EAAE,aAAa,CACrH,CAAC;YACF,MAAM,sBAAsB,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;QACL,IAAI,CAAC,IAAI;YAAE,SAAS;QAEpB,MAAM,eAAe,GAAG,QAAQ,CAC9B,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,eAAe,EAAE,SAAS,EAAE,EAC5D,IAAI,CACL;aACE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YACrB,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC;YACxC,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;QACzB,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YACtB,MAAM,CAAC,KAAK,CACV,EAAE,GAAG,EAAE,EACP,gCAAgC,SAAS,IAAI,aAAa,CAAC,MAAM,SAAS,WAAW,CAAC,EAAE,aAAa,CACtG,CAAC;YACF,OAAO,sBAAsB,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEL,eAAe,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,CAAC,IAAI,CACT,OAAO,aAAa,CAAC,MAAM,qBAAqB,QAAQ,CAAC,SAAS,iEAAiE,CACpI,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,sBAAsB,CAC7B,WAA+B,EAC/B,GAAY;IAEZ,OAAO;QACL,MAAM,EAAE,EAAE;QACV,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;QAClB,EAAE,EAAE,WAAW,CAAC,EAAE;QAClB,UAAU,EAAE,WAAW,CAAC,YAAY;QACpC,MAAM,EAAE,QAAQ;QAChB,eAAe,EAAE,WAAW,CAAC,eAAe;KAC7C,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,QAAyB,EACzB,WAA+B,EAC/B,GAAY;IAEZ,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,GAAG,sBAAsB,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;IAC1E,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;AACxB,CAAC"}
|
|
@@ -2,5 +2,8 @@ export declare function exists(filePath: string): Promise<boolean>;
|
|
|
2
2
|
export declare function filesize(path: string): Promise<import("fs").Stats | {
|
|
3
3
|
size: number;
|
|
4
4
|
}>;
|
|
5
|
+
export declare function formatFileTimestamp(timestamp: Date): string;
|
|
6
|
+
export declare function hashFile(filePath: string): Promise<string>;
|
|
5
7
|
export declare function isNotFoundError(error: unknown): boolean;
|
|
8
|
+
export declare function normalizeFileTimestamp(timestamp: Date): Date;
|
|
6
9
|
//# sourceMappingURL=filesystem.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"filesystem.d.ts","sourceRoot":"","sources":["../../../../src/downloader/fs/filesystem.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"filesystem.d.ts","sourceRoot":"","sources":["../../../../src/downloader/fs/filesystem.ts"],"names":[],"mappings":"AAIA,wBAAsB,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAQ/D;AAED,wBAAsB,QAAQ,CAAC,IAAI,EAAE,MAAM;;GAM1C;AAED,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,IAAI,GAAG,MAAM,CAE3D;AAED,wBAAsB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAOhE;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAEvD;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,IAAI,GAAG,IAAI,CAE5D"}
|
|
@@ -1,15 +1,37 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { createReadStream } from "node:fs";
|
|
1
3
|
import { stat } from "node:fs/promises";
|
|
2
4
|
export async function exists(filePath) {
|
|
3
5
|
return stat(filePath)
|
|
4
6
|
.then(() => true)
|
|
5
|
-
.catch(() =>
|
|
7
|
+
.catch((error) => {
|
|
8
|
+
if (isNotFoundError(error))
|
|
9
|
+
return false;
|
|
10
|
+
throw error;
|
|
11
|
+
});
|
|
6
12
|
}
|
|
7
13
|
export async function filesize(path) {
|
|
8
|
-
return stat(path).catch(() =>
|
|
14
|
+
return stat(path).catch((error) => {
|
|
15
|
+
if (isNotFoundError(error))
|
|
16
|
+
return { size: 0 };
|
|
17
|
+
throw error;
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
export function formatFileTimestamp(timestamp) {
|
|
21
|
+
return normalizeFileTimestamp(timestamp).toISOString();
|
|
22
|
+
}
|
|
23
|
+
export async function hashFile(filePath) {
|
|
24
|
+
const hash = createHash("sha256");
|
|
25
|
+
for await (const chunk of createReadStream(filePath))
|
|
26
|
+
hash.update(chunk);
|
|
27
|
+
return hash.digest("hex");
|
|
9
28
|
}
|
|
10
29
|
export function isNotFoundError(error) {
|
|
11
30
|
return isNodeError(error) && error.code === "ENOENT";
|
|
12
31
|
}
|
|
32
|
+
export function normalizeFileTimestamp(timestamp) {
|
|
33
|
+
return new Date(Math.trunc(timestamp.getTime() / 1000) * 1000);
|
|
34
|
+
}
|
|
13
35
|
function isNodeError(error) {
|
|
14
36
|
return error instanceof Error && "code" in error;
|
|
15
37
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"filesystem.js","sourceRoot":"","sources":["../../../../src/downloader/fs/filesystem.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAExC,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,QAAgB;IAC3C,OAAO,IAAI,CAAC,QAAQ,CAAC;SAClB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"filesystem.js","sourceRoot":"","sources":["../../../../src/downloader/fs/filesystem.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAExC,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,QAAgB;IAC3C,OAAO,IAAI,CAAC,QAAQ,CAAC;SAClB,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;QACxB,IAAI,eAAe,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAEzC,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;QACzC,IAAI,eAAe,CAAC,KAAK,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QAE/C,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,SAAe;IACjD,OAAO,sBAAsB,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,QAAgB;IAC7C,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAElC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,gBAAgB,CAAC,QAAQ,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,KAAe,CAAC,CAAC;IAE/B,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAc;IAC5C,OAAO,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,SAAe;IACpD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,IAAK,CAAC,GAAG,IAAK,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,CAAC;AACnD,CAAC"}
|
|
@@ -5,5 +5,5 @@ export interface RunCliDependencies {
|
|
|
5
5
|
logger: Logger;
|
|
6
6
|
transport?: HttpTransport;
|
|
7
7
|
}
|
|
8
|
-
export declare function download({ logger, transport: customTransport }: RunCliDependencies, options: DownloadOptions): Promise<
|
|
8
|
+
export declare function download({ logger, transport: customTransport }: RunCliDependencies, options: DownloadOptions): Promise<boolean>;
|
|
9
9
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/downloader/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAInC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/downloader/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAInC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,aAAa,CAAC;CAC3B;AAED,wBAAsB,QAAQ,CAC5B,EAAE,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,EAAE,kBAAkB,EAC1D,OAAO,EAAE,eAAe,oBA2EzB"}
|
|
@@ -27,7 +27,6 @@ export async function download({ logger, transport: customTransport }, options)
|
|
|
27
27
|
rootPath: options.output,
|
|
28
28
|
});
|
|
29
29
|
const client = new FanboxClient({ headers, transport });
|
|
30
|
-
const failed = [];
|
|
31
30
|
const creatorIds = await resolveCreatorIds({ client, logger }, options);
|
|
32
31
|
const creatorManifestManager = new CreatorManifestManager({
|
|
33
32
|
logger,
|
|
@@ -37,32 +36,30 @@ export async function download({ logger, transport: customTransport }, options)
|
|
|
37
36
|
for (const creatorId of creatorIds) {
|
|
38
37
|
logger.debug(`Initializing download for creator ${creatorId}`);
|
|
39
38
|
const creatorManifest = await creatorManifestManager.load(creatorId);
|
|
40
|
-
const creatorPathManager = pathManager.dir([
|
|
41
|
-
{ context: creatorId, required: true },
|
|
42
|
-
]);
|
|
43
39
|
const syncCreatorPromise = syncCreator({
|
|
44
40
|
client,
|
|
45
41
|
headers,
|
|
46
42
|
logger,
|
|
47
43
|
manifest: creatorManifest,
|
|
48
|
-
pathManager:
|
|
44
|
+
pathManager: pathManager.dir([{ context: creatorId, required: true }]),
|
|
49
45
|
transport,
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
46
|
+
verify: options.verify,
|
|
47
|
+
})
|
|
48
|
+
.then(() => {
|
|
49
|
+
creatorManifestManager.markSucceeded(creatorId);
|
|
50
|
+
})
|
|
51
|
+
.catch((err) => {
|
|
52
|
+
logger.error({ err }, `Error occurred while syncing creator ${creatorId}, skipping.`);
|
|
53
|
+
creatorManifestManager.markFailed(creatorId, err);
|
|
53
54
|
});
|
|
54
55
|
processingCreators.push(syncCreatorPromise);
|
|
55
|
-
const success = !hasFailures(creatorManifest);
|
|
56
|
-
if (!success)
|
|
57
|
-
failed.push(creatorId);
|
|
58
56
|
}
|
|
59
57
|
logger.info(`All ${creatorIds.length} creators have been initialized for download, waiting for all to complete...`);
|
|
60
58
|
await Promise.all(processingCreators);
|
|
61
59
|
await creatorManifestManager.saveAll();
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
return Object.values(manifest.posts).some((post) => post?.status === "failed");
|
|
60
|
+
const failedCreatorIds = creatorManifestManager.getFailedCreatorIds();
|
|
61
|
+
logger.info({ failedCreatorIds }, `Download completed for ${creatorIds.length} creators, with ${failedCreatorIds.length} failures.`);
|
|
62
|
+
const hasFailedCreators = failedCreatorIds.length > 0;
|
|
63
|
+
return hasFailedCreators;
|
|
67
64
|
}
|
|
68
65
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/downloader/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AAEzE,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/downloader/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAC;AAEzE,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AAOvE,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,EAAE,MAAM,EAAE,SAAS,EAAE,eAAe,EAAsB,EAC1D,OAAwB;IAExB,MAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,EAAE,gCAAgC,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,0BAA0B,CAAC;QACzC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;IACH,MAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,EAAE,6BAA6B,CAAC,CAAC;IACzD,MAAM,SAAS,GAAG,IAAI,aAAa,CACjC,EAAE,MAAM,EAAE,EACV;QACE,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,sBAAsB,EAAE,OAAO,CAAC,sBAAsB;QACtD,UAAU,EAAE,OAAO,CAAC,iBAAiB;QACrC,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,SAAS,EAAE,eAAe;KAC3B,CACF,CAAC;IACF,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC;QAClC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,QAAQ,EAAE,OAAO,CAAC,MAAM;KACzB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;IAExD,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,CAAC;IACxE,MAAM,sBAAsB,GAAG,IAAI,sBAAsB,CAAC;QACxD,MAAM;QACN,WAAW;KACZ,CAAC,CAAC;IACH,MAAM,kBAAkB,GAAoB,EAAE,CAAC;IAE/C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,CAAC,KAAK,CAAC,qCAAqC,SAAS,EAAE,CAAC,CAAC;QAC/D,MAAM,eAAe,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAErE,MAAM,kBAAkB,GAAG,WAAW,CAAC;YACrC,MAAM;YACN,OAAO;YACP,MAAM;YACN,QAAQ,EAAE,eAAe;YACzB,WAAW,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;YACtE,SAAS;YACT,MAAM,EAAE,OAAO,CAAC,MAAM;SACvB,CAAC;aACC,IAAI,CAAC,GAAG,EAAE;YACT,sBAAsB,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAClD,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YACtB,MAAM,CAAC,KAAK,CACV,EAAE,GAAG,EAAE,EACP,wCAAwC,SAAS,aAAa,CAC/D,CAAC;YACF,sBAAsB,CAAC,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;QACL,kBAAkB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,CAAC,IAAI,CACT,OAAO,UAAU,CAAC,MAAM,8EAA8E,CACvG,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IACtC,MAAM,sBAAsB,CAAC,OAAO,EAAE,CAAC;IACvC,MAAM,gBAAgB,GAAG,sBAAsB,CAAC,mBAAmB,EAAE,CAAC;IAEtE,MAAM,CAAC,IAAI,CACT,EAAE,gBAAgB,EAAE,EACpB,0BAA0B,UAAU,CAAC,MAAM,mBAAmB,gBAAgB,CAAC,MAAM,YAAY,CAClG,CAAC;IAEF,MAAM,iBAAiB,GAAG,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC;IACtD,OAAO,iBAAiB,CAAC;AAC3B,CAAC"}
|
|
@@ -8,7 +8,10 @@ interface CreatorManifestManagerDeps {
|
|
|
8
8
|
export declare class CreatorManifestManager {
|
|
9
9
|
#private;
|
|
10
10
|
constructor({ logger, pathManager }: CreatorManifestManagerDeps);
|
|
11
|
+
getFailedCreatorIds(): string[];
|
|
11
12
|
load(creatorId: string): Promise<CreatorManifest>;
|
|
13
|
+
markFailed(creatorId: string, error: unknown): void;
|
|
14
|
+
markSucceeded(creatorId: string): void;
|
|
12
15
|
saveAll(): Promise<void>;
|
|
13
16
|
}
|
|
14
17
|
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"creator-manager.d.ts","sourceRoot":"","sources":["../../../../src/downloader/manifest/creator-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C,UAAU,0BAA0B;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,WAAW,CAAC;CAC1B;AAED,qBAAa,sBAAsB;;gBAKrB,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,0BAA0B;
|
|
1
|
+
{"version":3,"file":"creator-manager.d.ts","sourceRoot":"","sources":["../../../../src/downloader/manifest/creator-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C,UAAU,0BAA0B;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,WAAW,CAAC;CAC1B;AAED,qBAAa,sBAAsB;;gBAKrB,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,0BAA0B;IAK/D,mBAAmB,IAAI,MAAM,EAAE;IASzB,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IASvD,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI;IAInD,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAIhC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CA2B/B"}
|