@moontaiworks/fanbox-dl 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -37
- package/dist/cjs/downloader/asset/download.d.ts +3 -1
- package/dist/cjs/downloader/asset/download.d.ts.map +1 -1
- package/dist/cjs/downloader/asset/download.js +11 -14
- package/dist/cjs/downloader/asset/download.js.map +1 -1
- package/dist/cjs/downloader/cli/options.d.ts +2 -1
- package/dist/cjs/downloader/cli/options.d.ts.map +1 -1
- package/dist/cjs/downloader/cli/options.js +3 -1
- package/dist/cjs/downloader/cli/options.js.map +1 -1
- package/dist/cjs/downloader/creator/discover-posts.d.ts.map +1 -1
- package/dist/cjs/downloader/creator/discover-posts.js +5 -3
- package/dist/cjs/downloader/creator/discover-posts.js.map +1 -1
- package/dist/cjs/downloader/creator/sync.d.ts +2 -1
- package/dist/cjs/downloader/creator/sync.d.ts.map +1 -1
- package/dist/cjs/downloader/creator/sync.js +25 -26
- package/dist/cjs/downloader/creator/sync.js.map +1 -1
- package/dist/cjs/downloader/fs/filesystem.d.ts +3 -0
- package/dist/cjs/downloader/fs/filesystem.d.ts.map +1 -1
- package/dist/cjs/downloader/fs/filesystem.js +27 -2
- package/dist/cjs/downloader/fs/filesystem.js.map +1 -1
- package/dist/cjs/downloader/index.d.ts +1 -1
- package/dist/cjs/downloader/index.d.ts.map +1 -1
- package/dist/cjs/downloader/index.js +13 -16
- package/dist/cjs/downloader/index.js.map +1 -1
- package/dist/cjs/downloader/manifest/creator-manager.d.ts +3 -0
- package/dist/cjs/downloader/manifest/creator-manager.d.ts.map +1 -1
- package/dist/cjs/downloader/manifest/creator-manager.js +24 -1
- package/dist/cjs/downloader/manifest/creator-manager.js.map +1 -1
- package/dist/cjs/downloader/manifest/creator.d.ts +6 -0
- package/dist/cjs/downloader/manifest/creator.d.ts.map +1 -1
- package/dist/cjs/downloader/manifest/creator.js +11 -1
- package/dist/cjs/downloader/manifest/creator.js.map +1 -1
- package/dist/cjs/downloader/post/article.d.ts.map +1 -1
- package/dist/cjs/downloader/post/article.js +25 -22
- package/dist/cjs/downloader/post/article.js.map +1 -1
- package/dist/cjs/downloader/post/sync.d.ts +3 -2
- package/dist/cjs/downloader/post/sync.d.ts.map +1 -1
- package/dist/cjs/downloader/post/sync.js +123 -85
- package/dist/cjs/downloader/post/sync.js.map +1 -1
- package/dist/cjs/downloader/post/verify.d.ts +9 -0
- package/dist/cjs/downloader/post/verify.d.ts.map +1 -0
- package/dist/cjs/downloader/post/verify.js +64 -0
- package/dist/cjs/downloader/post/verify.js.map +1 -0
- package/dist/esm/downloader/asset/download.d.ts +3 -1
- package/dist/esm/downloader/asset/download.d.ts.map +1 -1
- package/dist/esm/downloader/asset/download.js +12 -15
- package/dist/esm/downloader/asset/download.js.map +1 -1
- package/dist/esm/downloader/cli/options.d.ts +2 -1
- package/dist/esm/downloader/cli/options.d.ts.map +1 -1
- package/dist/esm/downloader/cli/options.js +3 -1
- package/dist/esm/downloader/cli/options.js.map +1 -1
- package/dist/esm/downloader/creator/discover-posts.d.ts.map +1 -1
- package/dist/esm/downloader/creator/discover-posts.js +5 -3
- package/dist/esm/downloader/creator/discover-posts.js.map +1 -1
- package/dist/esm/downloader/creator/sync.d.ts +2 -1
- package/dist/esm/downloader/creator/sync.d.ts.map +1 -1
- package/dist/esm/downloader/creator/sync.js +25 -26
- package/dist/esm/downloader/creator/sync.js.map +1 -1
- package/dist/esm/downloader/fs/filesystem.d.ts +3 -0
- package/dist/esm/downloader/fs/filesystem.d.ts.map +1 -1
- package/dist/esm/downloader/fs/filesystem.js +24 -2
- package/dist/esm/downloader/fs/filesystem.js.map +1 -1
- package/dist/esm/downloader/index.d.ts +1 -1
- package/dist/esm/downloader/index.d.ts.map +1 -1
- package/dist/esm/downloader/index.js +13 -16
- package/dist/esm/downloader/index.js.map +1 -1
- package/dist/esm/downloader/manifest/creator-manager.d.ts +3 -0
- package/dist/esm/downloader/manifest/creator-manager.d.ts.map +1 -1
- package/dist/esm/downloader/manifest/creator-manager.js +24 -1
- package/dist/esm/downloader/manifest/creator-manager.js.map +1 -1
- package/dist/esm/downloader/manifest/creator.d.ts +6 -0
- package/dist/esm/downloader/manifest/creator.d.ts.map +1 -1
- package/dist/esm/downloader/manifest/creator.js +11 -1
- package/dist/esm/downloader/manifest/creator.js.map +1 -1
- package/dist/esm/downloader/post/article.d.ts.map +1 -1
- package/dist/esm/downloader/post/article.js +25 -22
- package/dist/esm/downloader/post/article.js.map +1 -1
- package/dist/esm/downloader/post/sync.d.ts +3 -2
- package/dist/esm/downloader/post/sync.d.ts.map +1 -1
- package/dist/esm/downloader/post/sync.js +123 -85
- package/dist/esm/downloader/post/sync.js.map +1 -1
- package/dist/esm/downloader/post/verify.d.ts +9 -0
- package/dist/esm/downloader/post/verify.d.ts.map +1 -0
- package/dist/esm/downloader/post/verify.js +61 -0
- package/dist/esm/downloader/post/verify.js.map +1 -0
- package/package.json +1 -1
|
@@ -9,7 +9,7 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
|
|
|
9
9
|
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
|
10
10
|
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
11
11
|
};
|
|
12
|
-
var _CreatorManifestManager_instances, _CreatorManifestManager_logger, _CreatorManifestManager_manifests, _CreatorManifestManager_pathManager, _CreatorManifestManager_createManifest;
|
|
12
|
+
var _CreatorManifestManager_instances, _CreatorManifestManager_logger, _CreatorManifestManager_manifests, _CreatorManifestManager_pathManager, _CreatorManifestManager_createManifest, _CreatorManifestManager_loadedManifest;
|
|
13
13
|
import { CreatorManifest } from "./creator.js";
|
|
14
14
|
export class CreatorManifestManager {
|
|
15
15
|
constructor({ logger, pathManager }) {
|
|
@@ -20,11 +20,25 @@ export class CreatorManifestManager {
|
|
|
20
20
|
__classPrivateFieldSet(this, _CreatorManifestManager_logger, logger, "f");
|
|
21
21
|
__classPrivateFieldSet(this, _CreatorManifestManager_pathManager, pathManager, "f");
|
|
22
22
|
}
|
|
23
|
+
getFailedCreatorIds() {
|
|
24
|
+
const failed = new Set();
|
|
25
|
+
for (const [creatorId, manifest] of __classPrivateFieldGet(this, _CreatorManifestManager_manifests, "f")) {
|
|
26
|
+
if (manifest.error || hasFailedPost(manifest))
|
|
27
|
+
failed.add(creatorId);
|
|
28
|
+
}
|
|
29
|
+
return [...failed];
|
|
30
|
+
}
|
|
23
31
|
async load(creatorId) {
|
|
24
32
|
const manifest = __classPrivateFieldGet(this, _CreatorManifestManager_manifests, "f").get(creatorId) ?? __classPrivateFieldGet(this, _CreatorManifestManager_instances, "m", _CreatorManifestManager_createManifest).call(this, creatorId);
|
|
25
33
|
await manifest.load();
|
|
26
34
|
return manifest;
|
|
27
35
|
}
|
|
36
|
+
markFailed(creatorId, error) {
|
|
37
|
+
__classPrivateFieldGet(this, _CreatorManifestManager_instances, "m", _CreatorManifestManager_loadedManifest).call(this, creatorId).markFailed(error);
|
|
38
|
+
}
|
|
39
|
+
markSucceeded(creatorId) {
|
|
40
|
+
__classPrivateFieldGet(this, _CreatorManifestManager_instances, "m", _CreatorManifestManager_loadedManifest).call(this, creatorId).markSucceeded();
|
|
41
|
+
}
|
|
28
42
|
async saveAll() {
|
|
29
43
|
const manifests = Array.from(__classPrivateFieldGet(this, _CreatorManifestManager_manifests, "f").values());
|
|
30
44
|
await Promise.all(manifests.map((manifest) => manifest.save()));
|
|
@@ -39,5 +53,14 @@ _CreatorManifestManager_logger = new WeakMap(), _CreatorManifestManager_manifest
|
|
|
39
53
|
}, creatorId);
|
|
40
54
|
__classPrivateFieldGet(this, _CreatorManifestManager_manifests, "f").set(creatorId, manifest);
|
|
41
55
|
return manifest;
|
|
56
|
+
}, _CreatorManifestManager_loadedManifest = function _CreatorManifestManager_loadedManifest(creatorId) {
|
|
57
|
+
const manifest = __classPrivateFieldGet(this, _CreatorManifestManager_manifests, "f").get(creatorId);
|
|
58
|
+
if (!manifest) {
|
|
59
|
+
throw new Error(`Creator manifest ${creatorId} has not been loaded`);
|
|
60
|
+
}
|
|
61
|
+
return manifest;
|
|
42
62
|
};
|
|
63
|
+
function hasFailedPost(manifest) {
|
|
64
|
+
return Object.values(manifest.posts).some((post) => post?.status === "failed");
|
|
65
|
+
}
|
|
43
66
|
//# sourceMappingURL=creator-manager.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"creator-manager.js","sourceRoot":"","sources":["../../../../src/downloader/manifest/creator-manager.ts"],"names":[],"mappings":";;;;;;;;;;;;AAGA,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAO/C,MAAM,OAAO,sBAAsB;IAKjC,YAAY,EAAE,MAAM,EAAE,WAAW,EAA8B;;QAJ/D,iDAAgB;QAChB,4CAAa,IAAI,GAAG,EAA2B,EAAC;QAChD,sDAA0B;QAGxB,uBAAA,IAAI,kCAAW,MAAM,MAAA,CAAC;QACtB,uBAAA,IAAI,uCAAgB,WAAW,MAAA,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,SAAiB;QAC1B,MAAM,QAAQ,GACZ,uBAAA,IAAI,yCAAW,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,uBAAA,IAAI,iFAAgB,MAApB,IAAI,EAAiB,SAAS,CAAC,CAAC;QAEpE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAEtB,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,uBAAA,IAAI,yCAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QACvD,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClE,CAAC;
|
|
1
|
+
{"version":3,"file":"creator-manager.js","sourceRoot":"","sources":["../../../../src/downloader/manifest/creator-manager.ts"],"names":[],"mappings":";;;;;;;;;;;;AAGA,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAO/C,MAAM,OAAO,sBAAsB;IAKjC,YAAY,EAAE,MAAM,EAAE,WAAW,EAA8B;;QAJ/D,iDAAgB;QAChB,4CAAa,IAAI,GAAG,EAA2B,EAAC;QAChD,sDAA0B;QAGxB,uBAAA,IAAI,kCAAW,MAAM,MAAA,CAAC;QACtB,uBAAA,IAAI,uCAAgB,WAAW,MAAA,CAAC;IAClC,CAAC;IAED,mBAAmB;QACjB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;QACjC,KAAK,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,uBAAA,IAAI,yCAAW,EAAE,CAAC;YACpD,IAAI,QAAQ,CAAC,KAAK,IAAI,aAAa,CAAC,QAAQ,CAAC;gBAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACvE,CAAC;QAED,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,SAAiB;QAC1B,MAAM,QAAQ,GACZ,uBAAA,IAAI,yCAAW,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,uBAAA,IAAI,iFAAgB,MAApB,IAAI,EAAiB,SAAS,CAAC,CAAC;QAEpE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAEtB,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,UAAU,CAAC,SAAiB,EAAE,KAAc;QAC1C,uBAAA,IAAI,iFAAgB,MAApB,IAAI,EAAiB,SAAS,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC;IAED,aAAa,CAAC,SAAiB;QAC7B,uBAAA,IAAI,iFAAgB,MAApB,IAAI,EAAiB,SAAS,CAAC,CAAC,aAAa,EAAE,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,uBAAA,IAAI,yCAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QACvD,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClE,CAAC;CAwBF;oSAtBiB,SAAiB;IAC/B,MAAM,QAAQ,GAAG,IAAI,eAAe,CAClC;QACE,MAAM,EAAE,uBAAA,IAAI,sCAAQ;QACpB,WAAW,EAAE,uBAAA,IAAI,2CAAa,CAAC,GAAG,CAAC;YACjC,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE;SACvC,CAAC;KACH,EACD,SAAS,CACV,CAAC;IACF,uBAAA,IAAI,yCAAW,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACzC,OAAO,QAAQ,CAAC;AAClB,CAAC,2FAEe,SAAiB;IAC/B,MAAM,QAAQ,GAAG,uBAAA,IAAI,yCAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAChD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,oBAAoB,SAAS,sBAAsB,CAAC,CAAC;IACvE,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAGH,SAAS,aAAa,CAAC,QAAyB;IAC9C,OAAO,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CACvC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,QAAQ,CACpC,CAAC;AACJ,CAAC"}
|
|
@@ -3,7 +3,9 @@ import type { PathManager } from "../fs/path-manager.js";
|
|
|
3
3
|
import { type Store } from "./store.js";
|
|
4
4
|
export interface AssetManifestData {
|
|
5
5
|
bytes?: number;
|
|
6
|
+
contentIndex?: number;
|
|
6
7
|
error?: string;
|
|
8
|
+
expectedTime?: string;
|
|
7
9
|
path: string;
|
|
8
10
|
sha256?: string;
|
|
9
11
|
status: AssetStatus;
|
|
@@ -11,6 +13,7 @@ export interface AssetManifestData {
|
|
|
11
13
|
}
|
|
12
14
|
export interface CreatorManifestData {
|
|
13
15
|
creatorId: string;
|
|
16
|
+
error?: string;
|
|
14
17
|
posts: Partial<Record<string, PostManifestData>>;
|
|
15
18
|
version: 1;
|
|
16
19
|
}
|
|
@@ -33,9 +36,12 @@ export declare class CreatorManifest implements CreatorManifestData {
|
|
|
33
36
|
#private;
|
|
34
37
|
readonly creatorId: string;
|
|
35
38
|
readonly version: 1;
|
|
39
|
+
get error(): string | undefined;
|
|
36
40
|
get posts(): Partial<Record<string, PostManifestData>>;
|
|
37
41
|
constructor(options: CreatorManifestDeps, creatorId: string);
|
|
38
42
|
load(): Promise<void>;
|
|
43
|
+
markFailed(error: unknown): void;
|
|
44
|
+
markSucceeded(): void;
|
|
39
45
|
save(): Promise<void>;
|
|
40
46
|
}
|
|
41
47
|
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"creator.d.ts","sourceRoot":"","sources":["../../../../src/downloader/manifest/creator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAmB,KAAK,KAAK,EAAE,MAAM,YAAY,CAAC;AAEzD,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,WAAW,CAAC;IACpB,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC;IACjD,OAAO,EAAE,CAAC,CAAC;CACZ;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAC;IACnD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,OAAO,CAAC;IACpB,MAAM,EAAE,UAAU,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,KAAK,WAAW,GAAG,UAAU,GAAG,QAAQ,GAAG,UAAU,GAAG,SAAS,CAAC;AAElE,UAAU,mBAAmB;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,WAAW,CAAC;IACzB,KAAK,CAAC,EAAE,KAAK,CAAC,mBAAmB,CAAC,CAAC;CACpC;AAED,KAAK,UAAU,GAAG,UAAU,GAAG,QAAQ,GAAG,SAAS,GAAG,SAAS,GAAG,SAAS,CAAC;AAE5E,qBAAa,eAAgB,YAAW,mBAAmB;;
|
|
1
|
+
{"version":3,"file":"creator.d.ts","sourceRoot":"","sources":["../../../../src/downloader/manifest/creator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAmB,KAAK,KAAK,EAAE,MAAM,YAAY,CAAC;AAEzD,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,WAAW,CAAC;IACpB,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC;IACjD,OAAO,EAAE,CAAC,CAAC;CACZ;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAC;IACnD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,OAAO,CAAC;IACpB,MAAM,EAAE,UAAU,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,KAAK,WAAW,GAAG,UAAU,GAAG,QAAQ,GAAG,UAAU,GAAG,SAAS,CAAC;AAElE,UAAU,mBAAmB;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,WAAW,CAAC;IACzB,KAAK,CAAC,EAAE,KAAK,CAAC,mBAAmB,CAAC,CAAC;CACpC;AAED,KAAK,UAAU,GAAG,UAAU,GAAG,QAAQ,GAAG,SAAS,GAAG,SAAS,GAAG,SAAS,CAAC;AAE5E,qBAAa,eAAgB,YAAW,mBAAmB;;aAkBvC,SAAS,EAAE,MAAM;IAjBnC,QAAQ,CAAC,OAAO,EAAG,CAAC,CAAU;IAC9B,IAAI,KAAK,uBAER;IAED,IAAI,KAAK,8CAER;gBASC,OAAO,EAAE,mBAAmB,EACZ,SAAS,EAAE,MAAM;IAS7B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAa3B,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAIhC,aAAa,IAAI,IAAI;IAIf,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAkB5B"}
|
|
@@ -12,6 +12,9 @@ var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (
|
|
|
12
12
|
var _CreatorManifest_data, _CreatorManifest_loaded, _CreatorManifest_logger, _CreatorManifest_manifestPath, _CreatorManifest_store;
|
|
13
13
|
import { FileSystemStore } from "./store.js";
|
|
14
14
|
export class CreatorManifest {
|
|
15
|
+
get error() {
|
|
16
|
+
return __classPrivateFieldGet(this, _CreatorManifest_data, "f").error;
|
|
17
|
+
}
|
|
15
18
|
get posts() {
|
|
16
19
|
return __classPrivateFieldGet(this, _CreatorManifest_data, "f").posts;
|
|
17
20
|
}
|
|
@@ -36,9 +39,16 @@ export class CreatorManifest {
|
|
|
36
39
|
const data = await __classPrivateFieldGet(this, _CreatorManifest_store, "f").load(__classPrivateFieldGet(this, _CreatorManifest_manifestPath, "f"));
|
|
37
40
|
__classPrivateFieldSet(this, _CreatorManifest_data, data ?? { creatorId: this.creatorId, posts: {}, version: 1 }, "f");
|
|
38
41
|
}
|
|
42
|
+
markFailed(error) {
|
|
43
|
+
__classPrivateFieldGet(this, _CreatorManifest_data, "f").error = String(error);
|
|
44
|
+
}
|
|
45
|
+
markSucceeded() {
|
|
46
|
+
delete __classPrivateFieldGet(this, _CreatorManifest_data, "f").error;
|
|
47
|
+
}
|
|
39
48
|
async save() {
|
|
40
49
|
const posts = Object.values(__classPrivateFieldGet(this, _CreatorManifest_data, "f").posts);
|
|
41
|
-
if (!
|
|
50
|
+
if (!__classPrivateFieldGet(this, _CreatorManifest_data, "f").error &&
|
|
51
|
+
(!posts.length || posts.every((post) => post?.status !== "complete"))) {
|
|
42
52
|
__classPrivateFieldGet(this, _CreatorManifest_logger, "f").trace(`No posts in creator manifest for ${this.creatorId}, skipping save.`);
|
|
43
53
|
return;
|
|
44
54
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"creator.js","sourceRoot":"","sources":["../../../../src/downloader/manifest/creator.ts"],"names":[],"mappings":";;;;;;;;;;;;AAGA,OAAO,EAAE,eAAe,EAAc,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"creator.js","sourceRoot":"","sources":["../../../../src/downloader/manifest/creator.ts"],"names":[],"mappings":";;;;;;;;;;;;AAGA,OAAO,EAAE,eAAe,EAAc,MAAM,YAAY,CAAC;AAuCzD,MAAM,OAAO,eAAe;IAE1B,IAAI,KAAK;QACP,OAAO,uBAAA,IAAI,6BAAM,CAAC,KAAK,CAAC;IAC1B,CAAC;IAED,IAAI,KAAK;QACP,OAAO,uBAAA,IAAI,6BAAM,CAAC,KAAK,CAAC;IAC1B,CAAC;IAQD,YACE,OAA4B,EACZ,SAAiB;QAAjB,cAAS,GAAT,SAAS,CAAQ;QAjB1B,YAAO,GAAG,CAAU,CAAC;QAS9B,wCAA2B;QAC3B,kCAAU,KAAK,EAAC;QAChB,0CAAgB;QAChB,gDAAsB;QACtB,yCAAmC;QAMjC,uBAAA,IAAI,2BAAW,OAAO,CAAC,MAAM,MAAA,CAAC;QAC9B,uBAAA,IAAI,iCAAiB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,eAAe,CAAC,MAAA,CAAC;QAC/D,uBAAA,IAAI,0BAAU,OAAO,CAAC,KAAK,IAAI,IAAI,eAAe,EAAuB,MAAA,CAAC;QAE1E,uBAAA,IAAI,yBAAS,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,MAAA,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,uBAAA,IAAI,+BAAQ;YAAE,OAAO;QACzB,uBAAA,IAAI,+BAAQ,CAAC,KAAK,CAChB,gCAAgC,IAAI,CAAC,SAAS,SAAS,uBAAA,IAAI,qCAAc,EAAE,CAC5E,CAAC;QAEF,uBAAA,IAAI,2BAAW,IAAI,MAAA,CAAC;QAEpB,MAAM,IAAI,GAAG,MAAM,uBAAA,IAAI,8BAAO,CAAC,IAAI,CAAC,uBAAA,IAAI,qCAAc,CAAC,CAAC;QAExD,uBAAA,IAAI,yBAAS,IAAI,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,MAAA,CAAC;IAC5E,CAAC;IAED,UAAU,CAAC,KAAc;QACvB,uBAAA,IAAI,6BAAM,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAED,aAAa;QACX,OAAO,uBAAA,IAAI,6BAAM,CAAC,KAAK,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,uBAAA,IAAI,6BAAM,CAAC,KAAK,CAAC,CAAC;QAC9C,IACE,CAAC,uBAAA,IAAI,6BAAM,CAAC,KAAK;YACjB,CAAC,CAAC,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,UAAU,CAAC,CAAC,EACrE,CAAC;YAED,uBAAA,IAAI,+BAAQ,CAAC,KAAK,CAChB,oCAAoC,IAAI,CAAC,SAAS,kBAAkB,CACrE,CAAC;YACF,OAAO;QACT,CAAC;QAED,uBAAA,IAAI,+BAAQ,CAAC,KAAK,CAChB,+BAA+B,IAAI,CAAC,SAAS,OAAO,uBAAA,IAAI,qCAAc,EAAE,CACzE,CAAC;QACF,MAAM,uBAAA,IAAI,8BAAO,CAAC,IAAI,CAAC,uBAAA,IAAI,qCAAc,EAAE,uBAAA,IAAI,6BAAM,CAAC,CAAC;IACzD,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"article.d.ts","sourceRoot":"","sources":["../../../../src/downloader/post/article.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAc5C,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"article.d.ts","sourceRoot":"","sources":["../../../../src/downloader/post/article.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAc5C,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,EAAE,CAIlE"}
|
|
@@ -3,30 +3,33 @@ import { handleEmbedBlock } from "./embed.js";
|
|
|
3
3
|
import { handleUrlEmbedBlock } from "./url-embed.js";
|
|
4
4
|
export function formatArticleContents(post) {
|
|
5
5
|
return post.body.blocks
|
|
6
|
-
.map((block) =>
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
return
|
|
26
|
-
}
|
|
27
|
-
|
|
6
|
+
.map((block) => formatArticleBlock(post, block))
|
|
7
|
+
.filter(isContent);
|
|
8
|
+
}
|
|
9
|
+
function formatArticleBlock(post, block) {
|
|
10
|
+
if (isArticleBlockType("image", block)) {
|
|
11
|
+
const image = post.body.imageMap[block.imageId];
|
|
12
|
+
return new ImageContent(image);
|
|
13
|
+
}
|
|
14
|
+
if (isArticleBlockType("file", block)) {
|
|
15
|
+
const file = post.body.fileMap[block.fileId];
|
|
16
|
+
return new FileContent(file);
|
|
17
|
+
}
|
|
18
|
+
if (isArticleBlockType("url_embed", block)) {
|
|
19
|
+
return handleUrlEmbedBlock(block, post.body.urlEmbedMap);
|
|
20
|
+
}
|
|
21
|
+
if (isArticleBlockType("embed", block)) {
|
|
22
|
+
return handleEmbedBlock(block, post.body.embedMap);
|
|
23
|
+
}
|
|
24
|
+
if (isArticleBlockType("p", block) || isArticleBlockType("header", block)) {
|
|
25
|
+
return new TextContent(block);
|
|
26
|
+
}
|
|
27
|
+
return null;
|
|
28
28
|
}
|
|
29
29
|
function isArticleBlockType(type, item) {
|
|
30
30
|
return item.type === type;
|
|
31
31
|
}
|
|
32
|
+
function isContent(content) {
|
|
33
|
+
return content !== null;
|
|
34
|
+
}
|
|
32
35
|
//# sourceMappingURL=article.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"article.js","sourceRoot":"","sources":["../../../../src/downloader/post/article.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAWrD,MAAM,UAAU,qBAAqB,CAAC,IAAiB;IACrD,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM;SACpB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;
|
|
1
|
+
{"version":3,"file":"article.js","sourceRoot":"","sources":["../../../../src/downloader/post/article.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAWrD,MAAM,UAAU,qBAAqB,CAAC,IAAiB;IACrD,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM;SACpB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;SAC/C,MAAM,CAAC,SAAS,CAAC,CAAC;AACvB,CAAC;AAED,SAAS,kBAAkB,CACzB,IAAiB,EACjB,KAAmB;IAEnB,IAAI,kBAAkB,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAChD,OAAO,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;IAED,IAAI,kBAAkB,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC7C,OAAO,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,IAAI,kBAAkB,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,CAAC;QAC3C,OAAO,mBAAmB,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,kBAAkB,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,CAAC;QACvC,OAAO,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,kBAAkB,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,kBAAkB,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAGD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CACzB,IAAO,EACP,IAAkB;IAElB,OAAO,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC;AAC5B,CAAC;AAED,SAAS,SAAS,CAAC,OAAuB;IACxC,OAAO,OAAO,KAAK,IAAI,CAAC;AAC1B,CAAC"}
|
|
@@ -5,7 +5,8 @@ import type { PathManager } from "../fs/path-manager.js";
|
|
|
5
5
|
import type { CreatorManifest, PostManifestData } from "../manifest/creator.js";
|
|
6
6
|
interface PreSyncPostCheckDeps {
|
|
7
7
|
logger: Logger;
|
|
8
|
-
manifest: CreatorManifest
|
|
8
|
+
manifest: Pick<CreatorManifest, "posts">;
|
|
9
|
+
verify?: boolean;
|
|
9
10
|
}
|
|
10
11
|
interface SyncPostDeps {
|
|
11
12
|
headers?: Record<string, string>;
|
|
@@ -13,7 +14,7 @@ interface SyncPostDeps {
|
|
|
13
14
|
pathManager: PathManager;
|
|
14
15
|
transport: HttpTransport;
|
|
15
16
|
}
|
|
16
|
-
export declare function preSyncPostCheck({ logger, manifest }: PreSyncPostCheckDeps, postSummary: PostSummary): PostManifestData
|
|
17
|
+
export declare function preSyncPostCheck({ logger, manifest, verify }: PreSyncPostCheckDeps, postSummary: PostSummary): Promise<PostManifestData>;
|
|
17
18
|
export declare function syncPost({ headers, logger, pathManager, transport }: SyncPostDeps, post: Post): Promise<PostManifestData>;
|
|
18
19
|
export {};
|
|
19
20
|
//# sourceMappingURL=sync.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../../../src/downloader/post/sync.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,EAEV,eAAe,EACf,gBAAgB,EACjB,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../../../src/downloader/post/sync.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,EAEV,eAAe,EACf,gBAAgB,EACjB,MAAM,wBAAwB,CAAC;AAsBhC,UAAU,oBAAoB;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;IACzC,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAmBD,UAAU,YAAY;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,aAAa,CAAC;CAC1B;AAED,wBAAsB,gBAAgB,CACpC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,oBAAoB,EAClD,WAAW,EAAE,WAAW,GACvB,OAAO,CAAC,gBAAgB,CAAC,CAkC3B;AAED,wBAAsB,QAAQ,CAC5B,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,YAAY,EACzD,IAAI,EAAE,IAAI,GACT,OAAO,CAAC,gBAAgB,CAAC,CA8C3B"}
|
|
@@ -4,7 +4,17 @@ import { downloadAsset } from "../asset/download.js";
|
|
|
4
4
|
import { formatFileAsset, formatImageAsset, formatTextContent, } from "../markdown/asset.js";
|
|
5
5
|
import { FileContent, ImageContent, TextContent } from "./content.js";
|
|
6
6
|
import { formatPostContents } from "./contents.js";
|
|
7
|
-
|
|
7
|
+
import { verifyCompletePost } from "./verify.js";
|
|
8
|
+
export async function preSyncPostCheck({ logger, manifest, verify }, postSummary) {
|
|
9
|
+
const existingPost = manifest.posts[postSummary.id];
|
|
10
|
+
if (existingPost?.status === "complete" &&
|
|
11
|
+
postSummary.updatedDatetime === existingPost.updatedDatetime) {
|
|
12
|
+
logger.debug(`Post ${postSummary.id} has no changes, skipping download.`);
|
|
13
|
+
if (verify) {
|
|
14
|
+
return verifyCompletePost({ logger }, existingPost, postSummary);
|
|
15
|
+
}
|
|
16
|
+
return existingPost;
|
|
17
|
+
}
|
|
8
18
|
if (postSummary.isRestricted) {
|
|
9
19
|
logger.debug(`Post ${postSummary.id} is restricted, skipping download.`);
|
|
10
20
|
return {
|
|
@@ -15,12 +25,6 @@ export function preSyncPostCheck({ logger, manifest }, postSummary) {
|
|
|
15
25
|
updatedDatetime: postSummary.updatedDatetime,
|
|
16
26
|
};
|
|
17
27
|
}
|
|
18
|
-
const existingPost = manifest.posts[postSummary.id];
|
|
19
|
-
if (existingPost?.status === "complete" &&
|
|
20
|
-
postSummary.updatedDatetime === existingPost.updatedDatetime) {
|
|
21
|
-
logger.debug(`Post ${postSummary.id} has no changes, skipping download.`);
|
|
22
|
-
return existingPost;
|
|
23
|
-
}
|
|
24
28
|
return {
|
|
25
29
|
assets: {},
|
|
26
30
|
id: postSummary.id,
|
|
@@ -34,84 +38,14 @@ export async function syncPost({ headers, logger, pathManager, transport }, post
|
|
|
34
38
|
const contents = formatPostContents({ logger }, post);
|
|
35
39
|
if (post.coverImageUrl)
|
|
36
40
|
contents.unshift(formatCoverImage(post));
|
|
37
|
-
let hasUnknownContent = false;
|
|
38
|
-
const assetManifestData = {};
|
|
39
|
-
const download = downloadAsset.bind(undefined, {
|
|
40
|
-
headers,
|
|
41
|
-
logger,
|
|
42
|
-
transport,
|
|
43
|
-
});
|
|
44
41
|
const totalDigits = contents.length.toString().length;
|
|
45
|
-
const results = await Promise.allSettled(contents.map(
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
const { bytes, sha256 } = await download({
|
|
53
|
-
destination,
|
|
54
|
-
fallbackDateTime: post.updatedDatetime,
|
|
55
|
-
mediaContent: content,
|
|
56
|
-
}).catch((err) => {
|
|
57
|
-
logger.error({ err }, `Error occurred while downloading image asset ${content.id}, skipping.`);
|
|
58
|
-
assetManifestData[content.id] = {
|
|
59
|
-
path: destination,
|
|
60
|
-
status: "failed",
|
|
61
|
-
url: content.url,
|
|
62
|
-
};
|
|
63
|
-
throw err;
|
|
64
|
-
});
|
|
65
|
-
assetManifestData[content.id] = {
|
|
66
|
-
bytes,
|
|
67
|
-
path: destination,
|
|
68
|
-
sha256,
|
|
69
|
-
status: "complete",
|
|
70
|
-
url: content.url,
|
|
71
|
-
};
|
|
72
|
-
return formatImageAsset({ assetPath: destination });
|
|
73
|
-
}
|
|
74
|
-
if (content instanceof FileContent) {
|
|
75
|
-
const destination = pathManager.asset([
|
|
76
|
-
{ context: indexPadded, required: true },
|
|
77
|
-
{ context: content.name, required: false },
|
|
78
|
-
{ context: content.id, required: true },
|
|
79
|
-
], content.extension);
|
|
80
|
-
const { bytes, sha256 } = await download({
|
|
81
|
-
destination,
|
|
82
|
-
fallbackDateTime: post.updatedDatetime,
|
|
83
|
-
mediaContent: content,
|
|
84
|
-
}).catch((err) => {
|
|
85
|
-
logger.error({ err }, `Error occurred while downloading file asset ${content.id}, skipping.`);
|
|
86
|
-
assetManifestData[content.id] = {
|
|
87
|
-
path: destination,
|
|
88
|
-
status: "failed",
|
|
89
|
-
url: content.url,
|
|
90
|
-
};
|
|
91
|
-
throw err;
|
|
92
|
-
});
|
|
93
|
-
assetManifestData[content.id] = {
|
|
94
|
-
bytes,
|
|
95
|
-
path: destination,
|
|
96
|
-
sha256,
|
|
97
|
-
status: "complete",
|
|
98
|
-
url: content.url,
|
|
99
|
-
};
|
|
100
|
-
return formatFileAsset({ assetPath: destination });
|
|
101
|
-
}
|
|
102
|
-
if (content instanceof TextContent) {
|
|
103
|
-
return formatTextContent(content);
|
|
104
|
-
}
|
|
105
|
-
logger.warn(`Post ${post.id} has unhandled content type ${content.type}, skipping.`);
|
|
106
|
-
hasUnknownContent = true;
|
|
107
|
-
}));
|
|
108
|
-
const markdownContent = results.map((result) => {
|
|
109
|
-
if (result.status === "fulfilled") {
|
|
110
|
-
return result.value;
|
|
111
|
-
}
|
|
112
|
-
const err = String(result.reason);
|
|
113
|
-
return `<!-- Error occurred while processing content: ${err} -->`;
|
|
114
|
-
});
|
|
42
|
+
const results = await Promise.allSettled(contents.map((content, index) => processContent({ headers, logger, pathManager, post, transport }, { content, index, totalDigits })));
|
|
43
|
+
const hasRejectedContent = results.some((result) => result.status === "rejected");
|
|
44
|
+
const fulfilled = fulfilledResults(results);
|
|
45
|
+
const hasUnknownContent = fulfilled.some((result) => result.unknown);
|
|
46
|
+
const hasFailedContent = fulfilled.some((result) => result.failed);
|
|
47
|
+
const assetManifestData = collectAssetManifestData(fulfilled);
|
|
48
|
+
const markdownContent = formatMarkdownContent(results);
|
|
115
49
|
await writeFile(pathManager.asset([
|
|
116
50
|
{ context: "0".padStart(totalDigits, "0"), required: true },
|
|
117
51
|
{ context: "content", required: true },
|
|
@@ -120,10 +54,15 @@ export async function syncPost({ headers, logger, pathManager, transport }, post
|
|
|
120
54
|
assets: assetManifestData,
|
|
121
55
|
id: post.id,
|
|
122
56
|
restricted: false,
|
|
123
|
-
status: hasUnknownContent
|
|
57
|
+
status: hasUnknownContent || hasFailedContent || hasRejectedContent
|
|
58
|
+
? "partial"
|
|
59
|
+
: "complete",
|
|
124
60
|
updatedDatetime: post.updatedDatetime,
|
|
125
61
|
};
|
|
126
62
|
}
|
|
63
|
+
function collectAssetManifestData(contents) {
|
|
64
|
+
return Object.fromEntries(contents.flatMap((content) => content.assets ?? []));
|
|
65
|
+
}
|
|
127
66
|
function formatCoverImage(post) {
|
|
128
67
|
if (!post.coverImageUrl)
|
|
129
68
|
throw new Error("Post does not have a cover image");
|
|
@@ -134,4 +73,103 @@ function formatCoverImage(post) {
|
|
|
134
73
|
originalUrl: post.coverImageUrl,
|
|
135
74
|
});
|
|
136
75
|
}
|
|
76
|
+
function formatMarkdownContent(results) {
|
|
77
|
+
return results.map((result) => {
|
|
78
|
+
if (result.status === "fulfilled") {
|
|
79
|
+
return result.value.markdown;
|
|
80
|
+
}
|
|
81
|
+
const err = String(result.reason);
|
|
82
|
+
return `<!-- Error occurred while processing content: ${err} -->`;
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
function formatMediaAsset(content, destination) {
|
|
86
|
+
if (content instanceof ImageContent) {
|
|
87
|
+
return formatImageAsset({ assetPath: destination });
|
|
88
|
+
}
|
|
89
|
+
return formatFileAsset({ assetPath: destination });
|
|
90
|
+
}
|
|
91
|
+
function fulfilledResults(results) {
|
|
92
|
+
return results
|
|
93
|
+
.filter((result) => result.status === "fulfilled")
|
|
94
|
+
.map((result) => result.value);
|
|
95
|
+
}
|
|
96
|
+
function isMediaContent(content) {
|
|
97
|
+
return content instanceof FileContent || content instanceof ImageContent;
|
|
98
|
+
}
|
|
99
|
+
function mediaAssetSegments(content, index, totalDigits) {
|
|
100
|
+
const indexPadded = index.toString().padStart(totalDigits, "0");
|
|
101
|
+
if (content instanceof FileContent) {
|
|
102
|
+
return [
|
|
103
|
+
{ context: indexPadded, required: true },
|
|
104
|
+
{ context: content.name, required: false },
|
|
105
|
+
{ context: content.id, required: true },
|
|
106
|
+
];
|
|
107
|
+
}
|
|
108
|
+
return [
|
|
109
|
+
{ context: indexPadded, required: true },
|
|
110
|
+
{ context: content.id, required: true },
|
|
111
|
+
];
|
|
112
|
+
}
|
|
113
|
+
async function processContent({ headers, logger, pathManager, post, transport }, { content, index, totalDigits }) {
|
|
114
|
+
if (isMediaContent(content)) {
|
|
115
|
+
return syncMediaContent({ headers, logger, pathManager, transport }, {
|
|
116
|
+
content,
|
|
117
|
+
fallbackDateTime: post.updatedDatetime,
|
|
118
|
+
index,
|
|
119
|
+
totalDigits,
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
if (content instanceof TextContent) {
|
|
123
|
+
return { markdown: formatTextContent(content) };
|
|
124
|
+
}
|
|
125
|
+
logger.warn(`Post ${post.id} has unhandled content type ${content.type}, skipping.`);
|
|
126
|
+
return { unknown: true };
|
|
127
|
+
}
|
|
128
|
+
async function syncMediaContent({ headers, logger, pathManager, transport, }, { content, fallbackDateTime, index, totalDigits, }) {
|
|
129
|
+
const destination = pathManager.asset(mediaAssetSegments(content, index, totalDigits), content.extension);
|
|
130
|
+
const downloadResult = await downloadAsset({ headers, logger, transport }, {
|
|
131
|
+
destination,
|
|
132
|
+
fallbackDateTime,
|
|
133
|
+
mediaContent: content,
|
|
134
|
+
timeOffset: index,
|
|
135
|
+
})
|
|
136
|
+
.then(({ bytes, modifiedTime, sha256 }) => ({ bytes, failed: false, modifiedTime, sha256 }))
|
|
137
|
+
.catch((err) => {
|
|
138
|
+
logger.error({ err }, `Error occurred while downloading ${content.type} asset ${content.id}, skipping.`);
|
|
139
|
+
return { error: err, failed: true };
|
|
140
|
+
});
|
|
141
|
+
if (downloadResult.failed) {
|
|
142
|
+
return {
|
|
143
|
+
assets: [
|
|
144
|
+
[
|
|
145
|
+
content.id,
|
|
146
|
+
{
|
|
147
|
+
path: destination,
|
|
148
|
+
status: "failed",
|
|
149
|
+
url: content.url,
|
|
150
|
+
},
|
|
151
|
+
],
|
|
152
|
+
],
|
|
153
|
+
failed: true,
|
|
154
|
+
markdown: `<!-- Error occurred while processing content: ${String(downloadResult.error)} -->`,
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
return {
|
|
158
|
+
assets: [
|
|
159
|
+
[
|
|
160
|
+
content.id,
|
|
161
|
+
{
|
|
162
|
+
bytes: downloadResult.bytes,
|
|
163
|
+
contentIndex: index,
|
|
164
|
+
expectedTime: downloadResult.modifiedTime,
|
|
165
|
+
path: destination,
|
|
166
|
+
sha256: downloadResult.sha256,
|
|
167
|
+
status: "complete",
|
|
168
|
+
url: content.url,
|
|
169
|
+
},
|
|
170
|
+
],
|
|
171
|
+
],
|
|
172
|
+
markdown: formatMediaAsset(content, destination),
|
|
173
|
+
};
|
|
174
|
+
}
|
|
137
175
|
//# sourceMappingURL=sync.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../../../src/downloader/post/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAOrD,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../../../src/downloader/post/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAOrD,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AA2CjD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAwB,EAClD,WAAwB;IAExB,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IACpD,IACE,YAAY,EAAE,MAAM,KAAK,UAAU;QACnC,WAAW,CAAC,eAAe,KAAK,YAAY,CAAC,eAAe,EAC5D,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,QAAQ,WAAW,CAAC,EAAE,qCAAqC,CAAC,CAAC;QAC1E,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,kBAAkB,CAAC,EAAE,MAAM,EAAE,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;QACnE,CAAC;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,IAAI,WAAW,CAAC,YAAY,EAAE,CAAC;QAE7B,MAAM,CAAC,KAAK,CAAC,QAAQ,WAAW,CAAC,EAAE,oCAAoC,CAAC,CAAC;QACzE,OAAO;YACL,MAAM,EAAE,EAAE;YACV,EAAE,EAAE,WAAW,CAAC,EAAE;YAClB,UAAU,EAAE,IAAI;YAChB,MAAM,EAAE,SAAS;YACjB,eAAe,EAAE,WAAW,CAAC,eAAe;SAC7C,CAAC;IACJ,CAAC;IAED,OAAO;QACL,MAAM,EAAE,EAAE;QACV,EAAE,EAAE,WAAW,CAAC,EAAE;QAClB,UAAU,EAAE,KAAK;QACjB,MAAM,EAAE,SAAS;QACjB,eAAe,EAAE,WAAW,CAAC,eAAe;KAC7C,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAgB,EACzD,IAAU;IAEV,MAAM,CAAC,IAAI,CACT,oBAAoB,IAAI,CAAC,EAAE,eAAe,IAAI,CAAC,eAAe,eAAe,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,KAAK,EAAE,CAC7G,CAAC;IACF,MAAM,QAAQ,GAAG,kBAAkB,CAAC,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,CAAC;IACtD,IAAI,IAAI,CAAC,aAAa;QAAE,QAAQ,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;IAEjE,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC;IACtD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAC9B,cAAc,CACZ,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,EACjD,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,CAChC,CACF,CACF,CAAC;IACF,MAAM,kBAAkB,GAAG,OAAO,CAAC,IAAI,CACrC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,UAAU,CACzC,CAAC;IACF,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,iBAAiB,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACrE,MAAM,gBAAgB,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACnE,MAAM,iBAAiB,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC;IAC9D,MAAM,eAAe,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;IAEvD,MAAM,SAAS,CACb,WAAW,CAAC,KAAK,CACf;QACE,EAAE,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE;QAC3D,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE;KACvC,EACD,IAAI,CACL,EACD,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAC7B,CAAC;IAEF,OAAO;QACL,MAAM,EAAE,iBAAiB;QACzB,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,UAAU,EAAE,KAAK;QACjB,MAAM,EACJ,iBAAiB,IAAI,gBAAgB,IAAI,kBAAkB;YACzD,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,UAAU;QAChB,eAAe,EAAE,IAAI,CAAC,eAAe;KACtC,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAC/B,QAA4B;IAE5B,OAAO,MAAM,CAAC,WAAW,CACvB,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CACpD,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAU;IAClC,IAAI,CAAC,IAAI,CAAC,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAE7E,MAAM,SAAS,GACb,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;IAElE,OAAO,IAAI,YAAY,CAAC;QACtB,SAAS;QACT,EAAE,EAAE,OAAO;QACX,WAAW,EAAE,IAAI,CAAC,aAAa;KAChC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,qBAAqB,CAC5B,OAAiD;IAEjD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC5B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAClC,OAAO,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC/B,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClC,OAAO,iDAAiD,GAAG,MAAM,CAAC;IACpE,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAqB,EAAE,WAAmB;IAClE,IAAI,OAAO,YAAY,YAAY,EAAE,CAAC;QACpC,OAAO,gBAAgB,CAAC,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,eAAe,CAAC,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,gBAAgB,CACvB,OAAiD;IAEjD,OAAO,OAAO;SACX,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC;SACjD,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,cAAc,CACrB,OAAgB;IAEhB,OAAO,OAAO,YAAY,WAAW,IAAI,OAAO,YAAY,YAAY,CAAC;AAC3E,CAAC;AAED,SAAS,kBAAkB,CACzB,OAAqB,EACrB,KAAa,EACb,WAAmB;IAEnB,MAAM,WAAW,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;IAEhE,IAAI,OAAO,YAAY,WAAW,EAAE,CAAC;QACnC,OAAO;YACL,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE;YACxC,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE;YAC1C,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;SACxC,CAAC;IACJ,CAAC;IAED,OAAO;QACL,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE;QACxC,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;KACxC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAsB,EACrE,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAyB;IAEtD,IAAI,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,OAAO,gBAAgB,CACrB,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,EAC3C;YACE,OAAO;YACP,gBAAgB,EAAE,IAAI,CAAC,eAAe;YACtC,KAAK;YACL,WAAW;SACZ,CACF,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,YAAY,WAAW,EAAE,CAAC;QACnC,OAAO,EAAE,QAAQ,EAAE,iBAAiB,CAAC,OAAO,CAAC,EAAE,CAAC;IAClD,CAAC;IAED,MAAM,CAAC,IAAI,CACT,QAAQ,IAAI,CAAC,EAAE,+BAA+B,OAAO,CAAC,IAAI,aAAa,CACxE,CAAC;IACF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC3B,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,EACE,OAAO,EACP,MAAM,EACN,WAAW,EACX,SAAS,GAMV,EACD,EACE,OAAO,EACP,gBAAgB,EAChB,KAAK,EACL,WAAW,GAMZ;IAED,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CACnC,kBAAkB,CAAC,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,EAC/C,OAAO,CAAC,SAAS,CAClB,CAAC;IACF,MAAM,cAAc,GAAmB,MAAM,aAAa,CACxD,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,EAC9B;QACE,WAAW;QACX,gBAAgB;QAChB,YAAY,EAAE,OAAO;QACrB,UAAU,EAAE,KAAK;KAClB,CACF;SACE,IAAI,CACH,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE,CAClC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,CAAU,CAC5D;SACA,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;QACtB,MAAM,CAAC,KAAK,CACV,EAAE,GAAG,EAAE,EACP,oCAAoC,OAAO,CAAC,IAAI,UAAU,OAAO,CAAC,EAAE,aAAa,CAClF,CAAC;QAEF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAW,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEL,IAAI,cAAc,CAAC,MAAM,EAAE,CAAC;QAC1B,OAAO;YACL,MAAM,EAAE;gBACN;oBACE,OAAO,CAAC,EAAE;oBACV;wBACE,IAAI,EAAE,WAAW;wBACjB,MAAM,EAAE,QAAQ;wBAChB,GAAG,EAAE,OAAO,CAAC,GAAG;qBACjB;iBACF;aACF;YACD,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,iDAAiD,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM;SACnE,CAAC;IAC/B,CAAC;IAED,OAAO;QACL,MAAM,EAAE;YACN;gBACE,OAAO,CAAC,EAAE;gBACV;oBACE,KAAK,EAAE,cAAc,CAAC,KAAK;oBAC3B,YAAY,EAAE,KAAK;oBACnB,YAAY,EAAE,cAAc,CAAC,YAAY;oBACzC,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,cAAc,CAAC,MAAM;oBAC7B,MAAM,EAAE,UAAU;oBAClB,GAAG,EAAE,OAAO,CAAC,GAAG;iBACjB;aACF;SACF;QACD,QAAQ,EAAE,gBAAgB,CAAC,OAAO,EAAE,WAAW,CAAC;KACtB,CAAC;AAC/B,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Logger } from "pino";
|
|
2
|
+
import type { PostSummary } from "../../client/types.js";
|
|
3
|
+
import type { PostManifestData } from "../manifest/creator.js";
|
|
4
|
+
interface VerifyPostCheckDeps {
|
|
5
|
+
logger: Logger;
|
|
6
|
+
}
|
|
7
|
+
export declare function verifyCompletePost({ logger }: VerifyPostCheckDeps, manifest: PostManifestData, postSummary: PostSummary): Promise<PostManifestData>;
|
|
8
|
+
export {};
|
|
9
|
+
//# sourceMappingURL=verify.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"verify.d.ts","sourceRoot":"","sources":["../../../../src/downloader/post/verify.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAOzD,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,wBAAwB,CAAC;AAEhC,UAAU,mBAAmB;IAC3B,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,kBAAkB,CACtC,EAAE,MAAM,EAAE,EAAE,mBAAmB,EAC/B,QAAQ,EAAE,gBAAgB,EAC1B,WAAW,EAAE,WAAW,GACvB,OAAO,CAAC,gBAAgB,CAAC,CAqB3B"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { stat, utimes } from "node:fs/promises";
|
|
2
|
+
import { exists, formatFileTimestamp, hashFile, normalizeFileTimestamp, } from "../fs/filesystem.js";
|
|
3
|
+
export async function verifyCompletePost({ logger }, manifest, postSummary) {
|
|
4
|
+
const assets = Object.fromEntries(await Promise.all(Object.entries(manifest.assets).map(async ([id, asset]) => [
|
|
5
|
+
id,
|
|
6
|
+
asset ? await verifyAsset({ logger }, asset, postSummary) : asset,
|
|
7
|
+
])));
|
|
8
|
+
const hasFailedAsset = Object.values(assets).some((asset) => asset && asset.status !== "complete");
|
|
9
|
+
return {
|
|
10
|
+
...manifest,
|
|
11
|
+
assets,
|
|
12
|
+
status: hasFailedAsset ? "partial" : "complete",
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
function expectedModifiedTime(asset, postSummary) {
|
|
16
|
+
if (asset.expectedTime) {
|
|
17
|
+
const modifiedTime = normalizeFileTimestamp(new Date(asset.expectedTime));
|
|
18
|
+
if (!Number.isNaN(modifiedTime.getTime()))
|
|
19
|
+
return modifiedTime;
|
|
20
|
+
}
|
|
21
|
+
return normalizeFileTimestamp(new Date(new Date(postSummary.updatedDatetime).getTime() +
|
|
22
|
+
asset.contentIndex * 1000));
|
|
23
|
+
}
|
|
24
|
+
function failedAsset(asset, error) {
|
|
25
|
+
return {
|
|
26
|
+
...asset,
|
|
27
|
+
error,
|
|
28
|
+
status: "failed",
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
function obsoleteAsset(asset, error) {
|
|
32
|
+
return {
|
|
33
|
+
...asset,
|
|
34
|
+
error,
|
|
35
|
+
status: "obsolete",
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
async function verifyAsset({ logger }, asset, postSummary) {
|
|
39
|
+
if (asset.status !== "complete") {
|
|
40
|
+
return asset;
|
|
41
|
+
}
|
|
42
|
+
if (asset.contentIndex === undefined) {
|
|
43
|
+
return obsoleteAsset(asset, "content index missing");
|
|
44
|
+
}
|
|
45
|
+
if (!(await exists(asset.path))) {
|
|
46
|
+
return failedAsset(asset, "file missing");
|
|
47
|
+
}
|
|
48
|
+
const actualSha256 = await hashFile(asset.path);
|
|
49
|
+
if (actualSha256 !== asset.sha256) {
|
|
50
|
+
return failedAsset(asset, "sha256 mismatch");
|
|
51
|
+
}
|
|
52
|
+
const expectedTime = expectedModifiedTime(asset, postSummary);
|
|
53
|
+
const { mtimeMs } = await stat(asset.path);
|
|
54
|
+
if (normalizeFileTimestamp(new Date(mtimeMs)).getTime() !==
|
|
55
|
+
expectedTime.getTime()) {
|
|
56
|
+
logger.debug(`Repairing modified time of verified asset ${asset.path} from ${new Date(mtimeMs).toISOString()} to ${expectedTime.toISOString()}`);
|
|
57
|
+
await utimes(asset.path, expectedTime, expectedTime);
|
|
58
|
+
}
|
|
59
|
+
return { ...asset, expectedTime: formatFileTimestamp(expectedTime) };
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=verify.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"verify.js","sourceRoot":"","sources":["../../../../src/downloader/post/verify.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAKhD,OAAO,EACL,MAAM,EACN,mBAAmB,EACnB,QAAQ,EACR,sBAAsB,GACvB,MAAM,qBAAqB,CAAC;AAU7B,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,EAAE,MAAM,EAAuB,EAC/B,QAA0B,EAC1B,WAAwB;IAExB,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAC/B,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,CACjC,KAAK,EAAE,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CACpB;QACE,EAAE;QACF,KAAK,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK;KAChB,CACtD,CACF,CACF,CAAC;IACF,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAC/C,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU,CAChD,CAAC;IAEF,OAAO;QACL,GAAG,QAAQ;QACX,MAAM;QACN,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU;KAChD,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,KAAwB,EACxB,WAAwB;IAExB,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;QACvB,MAAM,YAAY,GAAG,sBAAsB,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC;QAC1E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;YAAE,OAAO,YAAY,CAAC;IACjE,CAAC;IAED,OAAO,sBAAsB,CAC3B,IAAI,IAAI,CACN,IAAI,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE;QAC7C,KAAK,CAAC,YAAa,GAAG,IAAK,CAC9B,CACF,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAClB,KAAwB,EACxB,KAAa;IAEb,OAAO;QACL,GAAG,KAAK;QACR,KAAK;QACL,MAAM,EAAE,QAAQ;KACjB,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,KAAwB,EACxB,KAAa;IAEb,OAAO;QACL,GAAG,KAAK;QACR,KAAK;QACL,MAAM,EAAE,UAAU;KACnB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,EAAE,MAAM,EAAuB,EAC/B,KAAwB,EACxB,WAAwB;IAExB,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACrC,OAAO,aAAa,CAAC,KAAK,EAAE,uBAAuB,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAChC,OAAO,WAAW,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChD,IAAI,YAAY,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QAClC,OAAO,WAAW,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAC9D,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,IACE,sBAAsB,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE;QACnD,YAAY,CAAC,OAAO,EAAE,EACtB,CAAC;QACD,MAAM,CAAC,KAAK,CACV,6CAA6C,KAAK,CAAC,IAAI,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,OAAO,YAAY,CAAC,WAAW,EAAE,EAAE,CACnI,CAAC;QACF,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC;IACvD,CAAC;IAED,OAAO,EAAE,GAAG,KAAK,EAAE,YAAY,EAAE,mBAAmB,CAAC,YAAY,CAAC,EAAE,CAAC;AACvE,CAAC"}
|