emdash-emit 0.1.14 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +96 -1
- package/README.md +259 -188
- package/dist/bin/emit.d.ts.map +1 -1
- package/dist/bin/emit.js +2 -0
- package/dist/bin/emit.js.map +1 -1
- package/dist/cli/commands/doctor.d.ts.map +1 -1
- package/dist/cli/commands/doctor.js +6 -0
- package/dist/cli/commands/doctor.js.map +1 -1
- package/dist/cli/commands/media.d.ts +4 -0
- package/dist/cli/commands/media.d.ts.map +1 -0
- package/dist/cli/commands/media.js +128 -0
- package/dist/cli/commands/media.js.map +1 -0
- package/dist/cli/commands/pull.d.ts +32 -0
- package/dist/cli/commands/pull.d.ts.map +1 -1
- package/dist/cli/commands/pull.js +136 -9
- package/dist/cli/commands/pull.js.map +1 -1
- package/dist/cli/commands/push.d.ts.map +1 -1
- package/dist/cli/commands/push.js +18 -0
- package/dist/cli/commands/push.js.map +1 -1
- package/dist/cli/commands/status.d.ts.map +1 -1
- package/dist/cli/commands/status.js +16 -5
- package/dist/cli/commands/status.js.map +1 -1
- package/dist/client/media.d.ts.map +1 -1
- package/dist/client/media.js +16 -5
- package/dist/client/media.js.map +1 -1
- package/dist/db/cache.d.ts +19 -4
- package/dist/db/cache.d.ts.map +1 -1
- package/dist/db/cache.js +127 -16
- package/dist/db/cache.js.map +1 -1
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +7 -3
- package/dist/mcp/server.js.map +1 -1
- package/dist/mcp/tools/sync-doctor.d.ts.map +1 -1
- package/dist/mcp/tools/sync-doctor.js +12 -0
- package/dist/mcp/tools/sync-doctor.js.map +1 -1
- package/dist/mcp/tools/sync-media-search.d.ts +4 -0
- package/dist/mcp/tools/sync-media-search.d.ts.map +1 -0
- package/dist/mcp/tools/sync-media-search.js +102 -0
- package/dist/mcp/tools/sync-media-search.js.map +1 -0
- package/dist/mcp/tools/sync-pull.d.ts.map +1 -1
- package/dist/mcp/tools/sync-pull.js +85 -13
- package/dist/mcp/tools/sync-pull.js.map +1 -1
- package/dist/mcp/tools/sync-push.d.ts.map +1 -1
- package/dist/mcp/tools/sync-push.js +5 -0
- package/dist/mcp/tools/sync-push.js.map +1 -1
- package/dist/mcp/tools/sync-status.d.ts.map +1 -1
- package/dist/mcp/tools/sync-status.js +6 -2
- package/dist/mcp/tools/sync-status.js.map +1 -1
- package/dist/sync/delete.d.ts +2 -1
- package/dist/sync/delete.d.ts.map +1 -1
- package/dist/sync/delete.js +6 -19
- package/dist/sync/delete.js.map +1 -1
- package/dist/sync/diff.d.ts.map +1 -1
- package/dist/sync/diff.js +18 -2
- package/dist/sync/diff.js.map +1 -1
- package/dist/sync/doctor.d.ts +27 -1
- package/dist/sync/doctor.d.ts.map +1 -1
- package/dist/sync/doctor.js +309 -26
- package/dist/sync/doctor.js.map +1 -1
- package/dist/sync/guards.d.ts +45 -0
- package/dist/sync/guards.d.ts.map +1 -0
- package/dist/sync/guards.js +236 -0
- package/dist/sync/guards.js.map +1 -0
- package/dist/sync/media/body_scan.d.ts +15 -0
- package/dist/sync/media/body_scan.d.ts.map +1 -0
- package/dist/sync/media/body_scan.js +132 -0
- package/dist/sync/media/body_scan.js.map +1 -0
- package/dist/sync/media/companions.d.ts +32 -0
- package/dist/sync/media/companions.d.ts.map +1 -0
- package/dist/sync/media/companions.js +260 -0
- package/dist/sync/media/companions.js.map +1 -0
- package/dist/sync/media/index.d.ts +9 -0
- package/dist/sync/media/index.d.ts.map +1 -0
- package/dist/sync/media/index.js +9 -0
- package/dist/sync/media/index.js.map +1 -0
- package/dist/sync/media/mirror_item.d.ts +27 -0
- package/dist/sync/media/mirror_item.d.ts.map +1 -0
- package/dist/sync/media/mirror_item.js +207 -0
- package/dist/sync/media/mirror_item.js.map +1 -0
- package/dist/sync/media/naming_mirror.d.ts +36 -0
- package/dist/sync/media/naming_mirror.d.ts.map +1 -0
- package/dist/sync/media/naming_mirror.js +143 -0
- package/dist/sync/media/naming_mirror.js.map +1 -0
- package/dist/sync/media/pull_article.d.ts +37 -0
- package/dist/sync/media/pull_article.d.ts.map +1 -0
- package/dist/sync/media/pull_article.js +273 -0
- package/dist/sync/media/pull_article.js.map +1 -0
- package/dist/sync/media/pull_library.d.ts +12 -0
- package/dist/sync/media/pull_library.d.ts.map +1 -0
- package/dist/sync/media/pull_library.js +255 -0
- package/dist/sync/media/pull_library.js.map +1 -0
- package/dist/sync/media/remote_index.d.ts +49 -0
- package/dist/sync/media/remote_index.d.ts.map +1 -0
- package/dist/sync/media/remote_index.js +241 -0
- package/dist/sync/media/remote_index.js.map +1 -0
- package/dist/sync/media/types.d.ts +237 -0
- package/dist/sync/media/types.d.ts.map +1 -0
- package/dist/sync/media/types.js +11 -0
- package/dist/sync/media/types.js.map +1 -0
- package/dist/sync/media.d.ts +1 -74
- package/dist/sync/media.d.ts.map +1 -1
- package/dist/sync/media.js +1 -592
- package/dist/sync/media.js.map +1 -1
- package/dist/sync/orchestrator.d.ts +23 -0
- package/dist/sync/orchestrator.d.ts.map +1 -1
- package/dist/sync/orchestrator.js +85 -1
- package/dist/sync/orchestrator.js.map +1 -1
- package/dist/sync/preflight.d.ts +1 -1
- package/dist/sync/preflight.d.ts.map +1 -1
- package/dist/sync/preflight.js +129 -17
- package/dist/sync/preflight.js.map +1 -1
- package/dist/sync/pull.d.ts +19 -0
- package/dist/sync/pull.d.ts.map +1 -1
- package/dist/sync/pull.js +169 -15
- package/dist/sync/pull.js.map +1 -1
- package/dist/sync/push.d.ts +7 -0
- package/dist/sync/push.d.ts.map +1 -1
- package/dist/sync/push.js +37 -10
- package/dist/sync/push.js.map +1 -1
- package/dist/sync/status.d.ts +4 -0
- package/dist/sync/status.d.ts.map +1 -1
- package/dist/sync/status.js +68 -4
- package/dist/sync/status.js.map +1 -1
- package/dist/utils/config.d.ts +25 -0
- package/dist/utils/config.d.ts.map +1 -1
- package/dist/utils/config.js +28 -2
- package/dist/utils/config.js.map +1 -1
- package/dist/utils/frontmatter.d.ts +1 -1
- package/dist/utils/frontmatter.d.ts.map +1 -1
- package/dist/utils/frontmatter.js +15 -4
- package/dist/utils/frontmatter.js.map +1 -1
- package/dist/utils/media-body-refs.d.ts +81 -0
- package/dist/utils/media-body-refs.d.ts.map +1 -0
- package/dist/utils/media-body-refs.js +240 -0
- package/dist/utils/media-body-refs.js.map +1 -0
- package/dist/utils/media-refs.d.ts +44 -0
- package/dist/utils/media-refs.d.ts.map +1 -0
- package/dist/utils/media-refs.js +101 -0
- package/dist/utils/media-refs.js.map +1 -0
- package/dist/utils/naming.d.ts +31 -0
- package/dist/utils/naming.d.ts.map +1 -1
- package/dist/utils/naming.js +58 -0
- package/dist/utils/naming.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import yaml from 'js-yaml';
|
|
4
|
+
import { formatDatePrefix, sanitizeTitleForFilename, storageKeyToMarkdownRef, collisionSuffixedMarkdownRef, } from '../../utils/frontmatter.js';
|
|
5
|
+
import { MEDIA_BINARY_EXTENSIONS } from './types.js';
|
|
6
|
+
/**
|
|
7
|
+
* Moves a file from src to dest, falling back to copy + unlink when src and dest
|
|
8
|
+
* reside on different filesystems (`EXDEV`).
|
|
9
|
+
*/
|
|
10
|
+
export function moveFileSafe(src, dest) {
|
|
11
|
+
try {
|
|
12
|
+
fs.renameSync(src, dest);
|
|
13
|
+
}
|
|
14
|
+
catch (err) {
|
|
15
|
+
if (err?.code === 'EXDEV') {
|
|
16
|
+
fs.copyFileSync(src, dest);
|
|
17
|
+
fs.unlinkSync(src);
|
|
18
|
+
}
|
|
19
|
+
else {
|
|
20
|
+
throw err;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Reads the media record id a mirror belongs to from its companion YAML, accepting both the storageKey-named
|
|
26
|
+
* stem and the legacy `<yyyyMMdd>_<id>-<name>` form. Returns `null` when no companion exists or it names no
|
|
27
|
+
* valid ULID, which is how Emit refuses to prune a file it cannot attribute to a record. A companion whose
|
|
28
|
+
* id is explicitly null (or any non-ULID value) proves the mirror backs no real record, so the stem's own
|
|
29
|
+
* leading characters are never trusted as an id — that fallback would let fabricated fixture ids reach
|
|
30
|
+
* record-keyed logic.
|
|
31
|
+
*/
|
|
32
|
+
export function mediaRecordIdForStem(mediaDir, stem) {
|
|
33
|
+
const candidates = [`${stem}.yaml`, `${stem.replace(/^\d{8}_/, '')}.yaml`];
|
|
34
|
+
for (const name of candidates) {
|
|
35
|
+
const candidatePath = path.join(mediaDir, name);
|
|
36
|
+
if (!fs.existsSync(candidatePath))
|
|
37
|
+
continue;
|
|
38
|
+
let doc = null;
|
|
39
|
+
try {
|
|
40
|
+
doc = yaml.load(fs.readFileSync(candidatePath, 'utf8'));
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
if (!doc)
|
|
46
|
+
continue;
|
|
47
|
+
const rawId = typeof doc.id === 'string' ? doc.id.trim() : '';
|
|
48
|
+
if (/^[0-9A-Za-z_-]{26}$/.test(rawId)) {
|
|
49
|
+
return rawId.toUpperCase();
|
|
50
|
+
}
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Removes any stale duplicate files in `media/` that belong to the same 26-char `mediaId` but sit under a
|
|
57
|
+
* different stem — a leftover date-prefixed name from an earlier local promotion, for instance. Ownership is
|
|
58
|
+
* read from each file's companion YAML rather than from the filename, because a storageKey-named mirror
|
|
59
|
+
* carries no record id at all and must never be matched (or deleted) by a coincidental ULID prefix.
|
|
60
|
+
*/
|
|
61
|
+
export function cleanupDuplicateMediaFilesForId(mediaDir, mediaId, canonicalBaseName) {
|
|
62
|
+
if (!fs.existsSync(mediaDir))
|
|
63
|
+
return;
|
|
64
|
+
const wanted = mediaId.toUpperCase();
|
|
65
|
+
for (const entry of fs.readdirSync(mediaDir)) {
|
|
66
|
+
if (entry.startsWith('legacy_'))
|
|
67
|
+
continue;
|
|
68
|
+
const entryBase = entry.replace(/\.yaml$/, '').replace(/\.[^.]+$/, '');
|
|
69
|
+
if (entryBase === canonicalBaseName)
|
|
70
|
+
continue;
|
|
71
|
+
// Only a companion-backed mirror can be attributed, and only when the companion itself is being
|
|
72
|
+
// retired together with its binary.
|
|
73
|
+
if (!entry.endsWith('.yaml'))
|
|
74
|
+
continue;
|
|
75
|
+
if (mediaRecordIdForStem(mediaDir, entryBase) !== wanted)
|
|
76
|
+
continue;
|
|
77
|
+
for (const orphan of [
|
|
78
|
+
`${entryBase}.yaml`,
|
|
79
|
+
...MEDIA_BINARY_EXTENSIONS.map((e) => `${entryBase}${e}`),
|
|
80
|
+
]) {
|
|
81
|
+
const orphanPath = path.join(mediaDir, orphan);
|
|
82
|
+
if (orphan === `${canonicalBaseName}.yaml`)
|
|
83
|
+
continue;
|
|
84
|
+
try {
|
|
85
|
+
if (fs.existsSync(orphanPath))
|
|
86
|
+
fs.unlinkSync(orphanPath);
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
// Ignore cleanup errors
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Resolves the storageKey-derived stem against the names already taken in `media/`. Two distinct keys can
|
|
96
|
+
* sanitize to the same basename — `a/sample.png` and `b/sample.png` both yield `sample` — and a second asset
|
|
97
|
+
* must never overwrite the first. A name already owned by this same record is reused as-is, so re-pulling is
|
|
98
|
+
* idempotent; any other collision gets the deterministic `<stem>--<mediaId>` suffix.
|
|
99
|
+
*/
|
|
100
|
+
export function resolveMarkdownRefCollision(mediaDir, baseRef, item) {
|
|
101
|
+
const suffixed = collisionSuffixedMarkdownRef(baseRef, item.id);
|
|
102
|
+
const takenByOther = (stem) => {
|
|
103
|
+
for (const suffix of ['.yaml', ...MEDIA_BINARY_EXTENSIONS]) {
|
|
104
|
+
const candidatePath = path.join(mediaDir, `${stem}${suffix}`);
|
|
105
|
+
if (!fs.existsSync(candidatePath))
|
|
106
|
+
continue;
|
|
107
|
+
const ownerId = mediaRecordIdForStem(mediaDir, stem);
|
|
108
|
+
if (ownerId && ownerId !== item.id.toUpperCase())
|
|
109
|
+
return true;
|
|
110
|
+
}
|
|
111
|
+
return false;
|
|
112
|
+
};
|
|
113
|
+
if (!takenByOther(baseRef))
|
|
114
|
+
return baseRef;
|
|
115
|
+
if (!takenByOther(suffixed))
|
|
116
|
+
return suffixed;
|
|
117
|
+
// Both taken and both owned by this record: the suffixed form is already this asset's, so keep it.
|
|
118
|
+
return suffixed;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Computes the canonical mirror name for one remote media item. Falls back to the record-id based stem only
|
|
122
|
+
* when the storageKey carries no usable basename, so a malformed migrated key cannot produce an unnameable
|
|
123
|
+
* file. Pass `mediaDir` to resolve a stem collision against the mirrors already on disk.
|
|
124
|
+
*/
|
|
125
|
+
export function mediaMirrorNames(item, mediaDir) {
|
|
126
|
+
const ext = path.extname(item.filename) || '';
|
|
127
|
+
const sanitizedBase = sanitizeTitleForFilename(path.basename(item.filename, ext));
|
|
128
|
+
const legacyBaseNames = [
|
|
129
|
+
`${item.id}-${sanitizedBase}`,
|
|
130
|
+
`${formatDatePrefix(item.createdAt)}${item.id}-${sanitizedBase}`,
|
|
131
|
+
];
|
|
132
|
+
const fromKey = item.storageKey ? storageKeyToMarkdownRef(item.storageKey) : null;
|
|
133
|
+
if (fromKey) {
|
|
134
|
+
const markdownRef = mediaDir ? resolveMarkdownRefCollision(mediaDir, fromKey, item) : fromKey;
|
|
135
|
+
return { markdownRef, ext, legacyBaseNames };
|
|
136
|
+
}
|
|
137
|
+
return {
|
|
138
|
+
markdownRef: `${formatDatePrefix(item.createdAt)}${item.id}-${sanitizedBase}`,
|
|
139
|
+
ext,
|
|
140
|
+
legacyBaseNames,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
//# sourceMappingURL=naming_mirror.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"naming_mirror.js","sourceRoot":"","sources":["../../../src/sync/media/naming_mirror.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,IAAI,MAAM,SAAS,CAAC;AAC3B,OAAO,EACL,gBAAgB,EAChB,wBAAwB,EACxB,uBAAuB,EACvB,4BAA4B,GAC7B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,uBAAuB,EAA+C,MAAM,YAAY,CAAC;AAElG;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,GAAW,EAAE,IAAY;IACpD,IAAI,CAAC;QACH,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,IAAK,GAA6B,EAAE,IAAI,KAAK,OAAO,EAAE,CAAC;YACrD,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YAC3B,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAAC,QAAgB,EAAE,IAAY;IACjE,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;IAC3E,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAChD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC;YAAE,SAAS;QAC5C,IAAI,GAAG,GAAmC,IAAI,CAAC;QAC/C,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,aAAa,EAAE,MAAM,CAAC,CAAmC,CAAC;QAC5F,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,MAAM,KAAK,GAAG,OAAO,GAAG,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,IAAI,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACtC,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;QAC7B,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,+BAA+B,CAC7C,QAAgB,EAChB,OAAe,EACf,iBAAyB;IAEzB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO;IACrC,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7C,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,SAAS;QAC1C,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QACvE,IAAI,SAAS,KAAK,iBAAiB;YAAE,SAAS;QAC9C,gGAAgG;QAChG,oCAAoC;QACpC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,SAAS;QACvC,IAAI,oBAAoB,CAAC,QAAQ,EAAE,SAAS,CAAC,KAAK,MAAM;YAAE,SAAS;QACnE,KAAK,MAAM,MAAM,IAAI;YACnB,GAAG,SAAS,OAAO;YACnB,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,SAAS,GAAG,CAAC,EAAE,CAAC;SAC1D,EAAE,CAAC;YACF,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC/C,IAAI,MAAM,KAAK,GAAG,iBAAiB,OAAO;gBAAE,SAAS;YACrD,IAAI,CAAC;gBACH,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC;oBAAE,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YAC3D,CAAC;YAAC,MAAM,CAAC;gBACP,wBAAwB;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,2BAA2B,CACzC,QAAgB,EAChB,OAAe,EACf,IAAqB;IAErB,MAAM,QAAQ,GAAG,4BAA4B,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG,CAAC,IAAY,EAAW,EAAE;QAC7C,KAAK,MAAM,MAAM,IAAI,CAAC,OAAO,EAAE,GAAG,uBAAuB,CAAC,EAAE,CAAC;YAC3D,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,IAAI,GAAG,MAAM,EAAE,CAAC,CAAC;YAC9D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC;gBAAE,SAAS;YAC5C,MAAM,OAAO,GAAG,oBAAoB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YACrD,IAAI,OAAO,IAAI,OAAO,KAAK,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE;gBAAE,OAAO,IAAI,CAAC;QAChE,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;QAAE,OAAO,OAAO,CAAC;IAC3C,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC7C,mGAAmG;IACnG,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAqB,EAAE,QAAiB;IACvE,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC9C,MAAM,aAAa,GAAG,wBAAwB,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC;IAClF,MAAM,eAAe,GAAG;QACtB,GAAG,IAAI,CAAC,EAAE,IAAI,aAAa,EAAE;QAC7B,GAAG,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,EAAE,IAAI,aAAa,EAAE;KACjE,CAAC;IAEF,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,uBAAuB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAClF,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,2BAA2B,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QAC9F,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,eAAe,EAAE,CAAC;IAC/C,CAAC;IACD,OAAO;QACL,WAAW,EAAE,GAAG,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,EAAE,IAAI,aAAa,EAAE;QAC7E,GAAG;QACH,eAAe;KAChB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { CacheDatabase } from '../../db/cache.js';
|
|
2
|
+
import type { MirroredAssetResult, PullMediaResult, RemoteStorageKeyIndex } from './types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Pulls media referenced by a specific article (by ULID or file path).
|
|
5
|
+
*/
|
|
6
|
+
export declare function pullArticleMedia(options: {
|
|
7
|
+
workspaceRoot: string;
|
|
8
|
+
origin: string;
|
|
9
|
+
token: string;
|
|
10
|
+
refreshToken?: string;
|
|
11
|
+
article: string;
|
|
12
|
+
collection?: string;
|
|
13
|
+
downloadFiles?: boolean;
|
|
14
|
+
fetchFn?: typeof fetch;
|
|
15
|
+
cache?: CacheDatabase;
|
|
16
|
+
}): Promise<PullMediaResult>;
|
|
17
|
+
/**
|
|
18
|
+
* Mirrors only the assets a batch of pulled articles actually references, reusing the records the lightweight
|
|
19
|
+
* index walk already fetched instead of paying for a per-asset `GET /_emdash/api/media/{id}`.
|
|
20
|
+
*
|
|
21
|
+
* This is the mirror half of the default-ON local-first pull: the walk phase builds `RemoteStorageKeyIndex`
|
|
22
|
+
* (JSON only, no binaries), and this phase downloads the binary + companion YAML for exactly the storageKeys
|
|
23
|
+
* found in article bodies. A key the walk did not report is NOT proof of a missing record — the record can
|
|
24
|
+
* exist under a different primary key while the body carries an R2 storageKey — so it is re-checked against
|
|
25
|
+
* confirmed evidence only (a walked record resolved under that exact ULID, or local mirror metadata) and what
|
|
26
|
+
* still fails is reported as `orphanStorageKeys`. Emit never fetches such an asset and never registers one.
|
|
27
|
+
*/
|
|
28
|
+
export declare function mirrorReferencedAssetsFromIndex(referencedStorageKeys: Iterable<string>, index: RemoteStorageKeyIndex, options: {
|
|
29
|
+
workspaceRoot: string;
|
|
30
|
+
origin: string;
|
|
31
|
+
token: string;
|
|
32
|
+
refreshToken?: string;
|
|
33
|
+
downloadFiles?: boolean;
|
|
34
|
+
fetchFn?: typeof fetch;
|
|
35
|
+
cache?: CacheDatabase;
|
|
36
|
+
}): Promise<MirroredAssetResult>;
|
|
37
|
+
//# sourceMappingURL=pull_article.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pull_article.d.ts","sourceRoot":"","sources":["../../../src/sync/media/pull_article.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAKvD,OAAO,KAAK,EAEV,mBAAmB,EAEnB,eAAe,EACf,qBAAqB,EACtB,MAAM,YAAY,CAAC;AAEpB;;GAEG;AACH,wBAAsB,gBAAgB,CAAC,OAAO,EAAE;IAC9C,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;IACvB,KAAK,CAAC,EAAE,aAAa,CAAC;CACvB,GAAG,OAAO,CAAC,eAAe,CAAC,CA6L3B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,+BAA+B,CACnD,qBAAqB,EAAE,QAAQ,CAAC,MAAM,CAAC,EACvC,KAAK,EAAE,qBAAqB,EAC5B,OAAO,EAAE;IACP,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;IACvB,KAAK,CAAC,EAAE,aAAa,CAAC;CACvB,GACA,OAAO,CAAC,mBAAmB,CAAC,CAuE9B"}
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { unwrapEnvelope } from '../../client/api.js';
|
|
4
|
+
import { ApiError } from '../../client/error.js';
|
|
5
|
+
import { createAuthedFetch } from '../../client/http.js';
|
|
6
|
+
import { extractMediaUlid } from '../../client/mapper.js';
|
|
7
|
+
import { parseMarkdown } from '../../utils/frontmatter.js';
|
|
8
|
+
import { collectArticleReferencedMediaIds, collectArticleStorageKeys } from './body_scan.js';
|
|
9
|
+
import { collectLocalMediaMirrorCandidates } from './companions.js';
|
|
10
|
+
import { syncSingleMediaItem } from './mirror_item.js';
|
|
11
|
+
/**
|
|
12
|
+
* Pulls media referenced by a specific article (by ULID or file path).
|
|
13
|
+
*/
|
|
14
|
+
export async function pullArticleMedia(options) {
|
|
15
|
+
const fetchFn = createAuthedFetch(options);
|
|
16
|
+
const collection = options.collection || 'posts';
|
|
17
|
+
const downloadFiles = options.downloadFiles !== false;
|
|
18
|
+
const mediaDir = path.join(options.workspaceRoot, 'media');
|
|
19
|
+
fs.mkdirSync(mediaDir, { recursive: true });
|
|
20
|
+
let articleId = options.article;
|
|
21
|
+
let articleContent = null;
|
|
22
|
+
// 1. Try resolving as direct file path
|
|
23
|
+
const directPath = path.isAbsolute(options.article)
|
|
24
|
+
? options.article
|
|
25
|
+
: path.resolve(options.workspaceRoot, options.article);
|
|
26
|
+
if (fs.existsSync(directPath) && fs.statSync(directPath).isFile()) {
|
|
27
|
+
articleContent = fs.readFileSync(directPath, 'utf8');
|
|
28
|
+
try {
|
|
29
|
+
const parsed = parseMarkdown(articleContent);
|
|
30
|
+
articleId = parsed.frontmatter.id || articleId;
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
// Keep articleId as fallback
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
else {
|
|
37
|
+
// 2. Try finding file in draft/published
|
|
38
|
+
for (const status of ['draft', 'published']) {
|
|
39
|
+
const statusDir = path.join(options.workspaceRoot, collection, status);
|
|
40
|
+
if (!fs.existsSync(statusDir))
|
|
41
|
+
continue;
|
|
42
|
+
const found = fs
|
|
43
|
+
.readdirSync(statusDir)
|
|
44
|
+
.find((f) => f.endsWith('.md') && f.includes(options.article));
|
|
45
|
+
if (found) {
|
|
46
|
+
articleContent = fs.readFileSync(path.join(statusDir, found), 'utf8');
|
|
47
|
+
try {
|
|
48
|
+
const parsed = parseMarkdown(articleContent);
|
|
49
|
+
articleId = parsed.frontmatter.id || articleId;
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
// fallback
|
|
53
|
+
}
|
|
54
|
+
break;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
// 3. If still absent locally, fetch remote article content from EmDash API
|
|
59
|
+
if (!articleContent) {
|
|
60
|
+
const url = new URL(`/_emdash/api/content/${collection}/${options.article}`, options.origin);
|
|
61
|
+
const res = await fetchFn(url.toString(), {
|
|
62
|
+
headers: {
|
|
63
|
+
Authorization: `Bearer ${options.token}`,
|
|
64
|
+
Accept: 'application/json',
|
|
65
|
+
},
|
|
66
|
+
});
|
|
67
|
+
if (!res.ok) {
|
|
68
|
+
throw new ApiError({
|
|
69
|
+
message: `Failed to fetch remote article for media resolution: ${res.statusText}`,
|
|
70
|
+
code: 'ARTICLE_NOT_FOUND',
|
|
71
|
+
status: res.status,
|
|
72
|
+
endpoint: url.toString(),
|
|
73
|
+
suggestion: `Verify that article '${options.article}' exists in collection '${collection}' on remote.`,
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
const json = await res.json();
|
|
77
|
+
const data = unwrapEnvelope(json);
|
|
78
|
+
const item = (data.item || data);
|
|
79
|
+
articleId = item.id || options.article;
|
|
80
|
+
// Build a content string from the remote representation so references can be collected even when
|
|
81
|
+
// no local file exists yet.
|
|
82
|
+
const remoteFi = item.featured_image || item.featuredImage;
|
|
83
|
+
const fiId = extractMediaUlid(remoteFi);
|
|
84
|
+
const fiPart = fiId ? `featuredImage: ${fiId}\n` : '';
|
|
85
|
+
const bodyPart = typeof item.content === 'string'
|
|
86
|
+
? item.content
|
|
87
|
+
: JSON.stringify(item.content || item.body || '');
|
|
88
|
+
articleContent = `---\n${fiPart}---\n${bodyPart}`;
|
|
89
|
+
}
|
|
90
|
+
const mediaIds = collectArticleReferencedMediaIds(articleContent);
|
|
91
|
+
// Body storageKeys are the only handle available when a migrated body bakes the R2 `storageKey`
|
|
92
|
+
// instead of the D1 record id. They must never be fed to `GET /_emdash/api/media/{id}` as if they
|
|
93
|
+
// were ids: that lookup keys on the primary key, so it returns 404 for a record that does exist.
|
|
94
|
+
const storageKeyDescriptors = collectArticleStorageKeys(articleContent);
|
|
95
|
+
const descriptorsByKey = new Map();
|
|
96
|
+
for (const descriptor of storageKeyDescriptors) {
|
|
97
|
+
descriptorsByKey.set(descriptor.storageKey, descriptor);
|
|
98
|
+
}
|
|
99
|
+
const allKeys = Array.from(descriptorsByKey.keys()).sort();
|
|
100
|
+
if (mediaIds.length === 0 && allKeys.length === 0) {
|
|
101
|
+
return {
|
|
102
|
+
total: 0,
|
|
103
|
+
downloaded: 0,
|
|
104
|
+
skipped: 0,
|
|
105
|
+
failed: 0,
|
|
106
|
+
articleId,
|
|
107
|
+
mediaIds: [],
|
|
108
|
+
orphanStorageKeys: [],
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
let downloadedCount = 0;
|
|
112
|
+
let skippedCount = 0;
|
|
113
|
+
// A body ULID that no canonical lookup resolves is the normal shape of a migrated corpus: the URL
|
|
114
|
+
// carries an R2 storageKey, not a media id. Only a reference that also survives every storageKey
|
|
115
|
+
// check is a problem, so a failed id lookup alone is never reported as `failed`.
|
|
116
|
+
let unresolvedIdCount = 0;
|
|
117
|
+
const recoveredStorageKeys = new Set();
|
|
118
|
+
// Ids whose canonical lookup succeeded. A body key sharing one of these ULIDs is a confirmed match:
|
|
119
|
+
// the record exists, it simply carries a storageKey that differs from the one baked into the body.
|
|
120
|
+
const resolvedRecordIds = new Set();
|
|
121
|
+
// 1. Canonical path: resolve each referenced media record id and mirror it normally.
|
|
122
|
+
for (const mId of mediaIds) {
|
|
123
|
+
try {
|
|
124
|
+
const mediaUrl = new URL(`/_emdash/api/media/${mId}`, options.origin);
|
|
125
|
+
const res = await fetchFn(mediaUrl.toString(), {
|
|
126
|
+
headers: {
|
|
127
|
+
Authorization: `Bearer ${options.token}`,
|
|
128
|
+
Accept: 'application/json',
|
|
129
|
+
},
|
|
130
|
+
});
|
|
131
|
+
if (!res.ok) {
|
|
132
|
+
unresolvedIdCount++;
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
const json = await res.json();
|
|
136
|
+
const envelope = unwrapEnvelope(json);
|
|
137
|
+
const item = (envelope.item || envelope);
|
|
138
|
+
const result = await syncSingleMediaItem({
|
|
139
|
+
item,
|
|
140
|
+
mediaDir,
|
|
141
|
+
origin: options.origin,
|
|
142
|
+
token: options.token,
|
|
143
|
+
downloadFiles,
|
|
144
|
+
fetchFn,
|
|
145
|
+
cache: options.cache,
|
|
146
|
+
});
|
|
147
|
+
if (result.downloaded)
|
|
148
|
+
downloadedCount++;
|
|
149
|
+
else if (result.skipped)
|
|
150
|
+
skippedCount++;
|
|
151
|
+
resolvedRecordIds.add(mId.toUpperCase());
|
|
152
|
+
if (item.storageKey)
|
|
153
|
+
recoveredStorageKeys.add(item.storageKey);
|
|
154
|
+
}
|
|
155
|
+
catch {
|
|
156
|
+
unresolvedIdCount++;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
// 2. Unresolved storageKeys: a body key that no canonical id lookup mirrored is NOT proof of a
|
|
160
|
+
// missing record. The record can exist under a different primary key, so Emit re-checks the key
|
|
161
|
+
// against confirmed evidence only — a record it just resolved under the same ULID, or the local
|
|
162
|
+
// mirror metadata (companion YAML `id` + `storageKey`) — and reports what still fails as
|
|
163
|
+
// unresolved. Emit never fetches such an asset and never registers a record for it.
|
|
164
|
+
const mirrorCandidates = collectLocalMediaMirrorCandidates(mediaDir);
|
|
165
|
+
const verifiedStorageKeys = [];
|
|
166
|
+
const orphanStorageKeys = [];
|
|
167
|
+
for (const key of allKeys) {
|
|
168
|
+
if (recoveredStorageKeys.has(key))
|
|
169
|
+
continue;
|
|
170
|
+
const ulids = (key.match(/[0-9A-HJ-NP-Z]{26}/gi) ?? []).map((u) => u.toUpperCase());
|
|
171
|
+
// Only two things count as evidence: a record Emit successfully fetched under this exact ULID, or
|
|
172
|
+
// confirmed local mirror metadata naming it. A body reference alone is never self-confirming — that
|
|
173
|
+
// is precisely the case where the asset must be reported instead of assumed healthy.
|
|
174
|
+
const confirmed = ulids.some((ulid) => resolvedRecordIds.has(ulid) || mirrorCandidates.has(ulid));
|
|
175
|
+
if (confirmed) {
|
|
176
|
+
verifiedStorageKeys.push(key);
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
orphanStorageKeys.push(key);
|
|
180
|
+
}
|
|
181
|
+
return {
|
|
182
|
+
total: mediaIds.length + verifiedStorageKeys.length + orphanStorageKeys.length,
|
|
183
|
+
downloaded: downloadedCount,
|
|
184
|
+
skipped: skippedCount,
|
|
185
|
+
failed: orphanStorageKeys.length,
|
|
186
|
+
unresolvedIds: unresolvedIdCount,
|
|
187
|
+
articleId,
|
|
188
|
+
mediaIds,
|
|
189
|
+
verifiedStorageKeys,
|
|
190
|
+
orphanStorageKeys,
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Mirrors only the assets a batch of pulled articles actually references, reusing the records the lightweight
|
|
195
|
+
* index walk already fetched instead of paying for a per-asset `GET /_emdash/api/media/{id}`.
|
|
196
|
+
*
|
|
197
|
+
* This is the mirror half of the default-ON local-first pull: the walk phase builds `RemoteStorageKeyIndex`
|
|
198
|
+
* (JSON only, no binaries), and this phase downloads the binary + companion YAML for exactly the storageKeys
|
|
199
|
+
* found in article bodies. A key the walk did not report is NOT proof of a missing record — the record can
|
|
200
|
+
* exist under a different primary key while the body carries an R2 storageKey — so it is re-checked against
|
|
201
|
+
* confirmed evidence only (a walked record resolved under that exact ULID, or local mirror metadata) and what
|
|
202
|
+
* still fails is reported as `orphanStorageKeys`. Emit never fetches such an asset and never registers one.
|
|
203
|
+
*/
|
|
204
|
+
export async function mirrorReferencedAssetsFromIndex(referencedStorageKeys, index, options) {
|
|
205
|
+
const fetchFn = createAuthedFetch(options);
|
|
206
|
+
const downloadFiles = options.downloadFiles !== false;
|
|
207
|
+
const mediaDir = path.join(options.workspaceRoot, 'media');
|
|
208
|
+
fs.mkdirSync(mediaDir, { recursive: true });
|
|
209
|
+
const mirroredStorageKeys = new Set();
|
|
210
|
+
let downloadedCount = 0;
|
|
211
|
+
let skippedCount = 0;
|
|
212
|
+
// Walk every referenced key once. A key resolves to a walked record either directly (the walk reported
|
|
213
|
+
// that exact storageKey) or through a ULID the walk resolved under that record id — the migrated-corpus
|
|
214
|
+
// case where the body URL bakes an R2 storageKey that differs from the record's own primary key.
|
|
215
|
+
const referenced = [...referencedStorageKeys];
|
|
216
|
+
const mirrorCandidates = collectLocalMediaMirrorCandidates(mediaDir);
|
|
217
|
+
const verifiedStorageKeys = [];
|
|
218
|
+
const orphanStorageKeys = [];
|
|
219
|
+
for (const key of referenced) {
|
|
220
|
+
let item = index.recordsByStorageKey.get(key);
|
|
221
|
+
if (!item) {
|
|
222
|
+
// The body key may itself be a record id the walk stored under `recordsById`, or share a ULID with a
|
|
223
|
+
// walked record. Reuse that record rather than re-fetching it.
|
|
224
|
+
const ulids = (key.match(/[0-9A-HJ-NP-Z]{26}/gi) ?? []).map((u) => u.toUpperCase());
|
|
225
|
+
for (const ulid of ulids) {
|
|
226
|
+
const byId = index.recordsById.get(ulid);
|
|
227
|
+
if (byId) {
|
|
228
|
+
item = byId;
|
|
229
|
+
break;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
if (item) {
|
|
234
|
+
const result = await syncSingleMediaItem({
|
|
235
|
+
item,
|
|
236
|
+
mediaDir,
|
|
237
|
+
origin: options.origin,
|
|
238
|
+
token: options.token,
|
|
239
|
+
downloadFiles,
|
|
240
|
+
fetchFn,
|
|
241
|
+
cache: options.cache,
|
|
242
|
+
});
|
|
243
|
+
if (result.downloaded)
|
|
244
|
+
downloadedCount++;
|
|
245
|
+
else if (result.skipped)
|
|
246
|
+
skippedCount++;
|
|
247
|
+
if (item.storageKey)
|
|
248
|
+
mirroredStorageKeys.add(item.storageKey);
|
|
249
|
+
mirroredStorageKeys.add(key);
|
|
250
|
+
continue;
|
|
251
|
+
}
|
|
252
|
+
// No walked record owns this key. Only two things count as evidence it is healthy: a local mirror
|
|
253
|
+
// companion already names the ULID, or the walk stored a matching storageKey. A bare body reference is
|
|
254
|
+
// never self-confirming — that is precisely the case to report rather than assume healthy.
|
|
255
|
+
const ulids = (key.match(/[0-9A-HJ-NP-Z]{26}/gi) ?? []).map((u) => u.toUpperCase());
|
|
256
|
+
const confirmed = index.storageKeys.has(key) ||
|
|
257
|
+
ulids.some((ulid) => index.recordsById.has(ulid) || mirrorCandidates.has(ulid));
|
|
258
|
+
if (confirmed) {
|
|
259
|
+
verifiedStorageKeys.push(key);
|
|
260
|
+
}
|
|
261
|
+
else {
|
|
262
|
+
orphanStorageKeys.push(key);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
return {
|
|
266
|
+
mirroredStorageKeys,
|
|
267
|
+
downloaded: downloadedCount,
|
|
268
|
+
skipped: skippedCount,
|
|
269
|
+
verifiedStorageKeys,
|
|
270
|
+
orphanStorageKeys,
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
//# sourceMappingURL=pull_article.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pull_article.js","sourceRoot":"","sources":["../../../src/sync/media/pull_article.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE1D,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,gCAAgC,EAAE,yBAAyB,EAAE,MAAM,gBAAgB,CAAC;AAC7F,OAAO,EAAE,iCAAiC,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AASvD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAUtC;IACC,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC;IACjD,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,KAAK,KAAK,CAAC;IACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAC3D,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5C,IAAI,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAChC,IAAI,cAAc,GAAkB,IAAI,CAAC;IAEzC,uCAAuC;IACvC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC;QACjD,CAAC,CAAC,OAAO,CAAC,OAAO;QACjB,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEzD,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;QAClE,cAAc,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,aAAa,CAAC,cAAc,CAAC,CAAC;YAC7C,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,SAAS,CAAC;QACjD,CAAC;QAAC,MAAM,CAAC;YACP,6BAA6B;QAC/B,CAAC;IACH,CAAC;SAAM,CAAC;QACN,yCAAyC;QACzC,KAAK,MAAM,MAAM,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,EAAE,CAAC;YAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;YACvE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;gBAAE,SAAS;YACxC,MAAM,KAAK,GAAG,EAAE;iBACb,WAAW,CAAC,SAAS,CAAC;iBACtB,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;YACjE,IAAI,KAAK,EAAE,CAAC;gBACV,cAAc,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC;gBACtE,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,aAAa,CAAC,cAAc,CAAC,CAAC;oBAC7C,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,SAAS,CAAC;gBACjD,CAAC;gBAAC,MAAM,CAAC;oBACP,WAAW;gBACb,CAAC;gBACD,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,wBAAwB,UAAU,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC7F,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YACxC,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,OAAO,CAAC,KAAK,EAAE;gBACxC,MAAM,EAAE,kBAAkB;aAC3B;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,QAAQ,CAAC;gBACjB,OAAO,EAAE,wDAAwD,GAAG,CAAC,UAAU,EAAE;gBACjF,IAAI,EAAE,mBAAmB;gBACzB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE;gBACxB,UAAU,EAAE,wBAAwB,OAAO,CAAC,OAAO,2BAA2B,UAAU,cAAc;aACvG,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,cAAc,CAA0B,IAAI,CAAC,CAAC;QAC3D,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAA4B,CAAC;QAC5D,SAAS,GAAI,IAAI,CAAC,EAAa,IAAI,OAAO,CAAC,OAAO,CAAC;QAEnD,iGAAiG;QACjG,4BAA4B;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,aAAa,CAAC;QAC3D,MAAM,IAAI,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,kBAAkB,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,MAAM,QAAQ,GACZ,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;YAC9B,CAAC,CAAC,IAAI,CAAC,OAAO;YACd,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QACtD,cAAc,GAAG,QAAQ,MAAM,QAAQ,QAAQ,EAAE,CAAC;IACpD,CAAC;IAED,MAAM,QAAQ,GAAG,gCAAgC,CAAC,cAAc,CAAC,CAAC;IAClE,gGAAgG;IAChG,kGAAkG;IAClG,iGAAiG;IACjG,MAAM,qBAAqB,GAAG,yBAAyB,CAAC,cAAc,CAAC,CAAC;IACxE,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAiC,CAAC;IAClE,KAAK,MAAM,UAAU,IAAI,qBAAqB,EAAE,CAAC;QAC/C,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAC1D,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAE3D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClD,OAAO;YACL,KAAK,EAAE,CAAC;YACR,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,CAAC;YACT,SAAS;YACT,QAAQ,EAAE,EAAE;YACZ,iBAAiB,EAAE,EAAE;SACtB,CAAC;IACJ,CAAC;IAED,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,kGAAkG;IAClG,iGAAiG;IACjG,iFAAiF;IACjF,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/C,oGAAoG;IACpG,mGAAmG;IACnG,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE5C,qFAAqF;IACrF,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,sBAAsB,GAAG,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YACtE,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE;gBAC7C,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,OAAO,CAAC,KAAK,EAAE;oBACxC,MAAM,EAAE,kBAAkB;iBAC3B;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,iBAAiB,EAAE,CAAC;gBACpB,SAAS;YACX,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAC9B,MAAM,QAAQ,GAAG,cAAc,CAA0B,IAAI,CAAC,CAAC;YAC/D,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAoB,CAAC;YAE5D,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC;gBACvC,IAAI;gBACJ,QAAQ;gBACR,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,aAAa;gBACb,OAAO;gBACP,KAAK,EAAE,OAAO,CAAC,KAAK;aACrB,CAAC,CAAC;YAEH,IAAI,MAAM,CAAC,UAAU;gBAAE,eAAe,EAAE,CAAC;iBACpC,IAAI,MAAM,CAAC,OAAO;gBAAE,YAAY,EAAE,CAAC;YACxC,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;YACzC,IAAI,IAAI,CAAC,UAAU;gBAAE,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACjE,CAAC;QAAC,MAAM,CAAC;YACP,iBAAiB,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;IAED,+FAA+F;IAC/F,mGAAmG;IACnG,mGAAmG;IACnG,4FAA4F;IAC5F,uFAAuF;IACvF,MAAM,gBAAgB,GAAG,iCAAiC,CAAC,QAAQ,CAAC,CAAC;IACrE,MAAM,mBAAmB,GAAa,EAAE,CAAC;IACzC,MAAM,iBAAiB,GAAa,EAAE,CAAC;IACvC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5C,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QACpF,kGAAkG;QAClG,oGAAoG;QACpG,qFAAqF;QACrF,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAC1B,CAAC,IAAI,EAAE,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,CACpE,CAAC;QACF,IAAI,SAAS,EAAE,CAAC;YACd,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9B,CAAC;IAED,OAAO;QACL,KAAK,EAAE,QAAQ,CAAC,MAAM,GAAG,mBAAmB,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM;QAC9E,UAAU,EAAE,eAAe;QAC3B,OAAO,EAAE,YAAY;QACrB,MAAM,EAAE,iBAAiB,CAAC,MAAM;QAChC,aAAa,EAAE,iBAAiB;QAChC,SAAS;QACT,QAAQ;QACR,mBAAmB;QACnB,iBAAiB;KAClB,CAAC;AACJ,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACnD,qBAAuC,EACvC,KAA4B,EAC5B,OAQC;IAED,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,KAAK,KAAK,CAAC;IACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAC3D,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5C,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9C,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,uGAAuG;IACvG,wGAAwG;IACxG,iGAAiG;IACjG,MAAM,UAAU,GAAG,CAAC,GAAG,qBAAqB,CAAC,CAAC;IAC9C,MAAM,gBAAgB,GAAG,iCAAiC,CAAC,QAAQ,CAAC,CAAC;IACrE,MAAM,mBAAmB,GAAa,EAAE,CAAC;IACzC,MAAM,iBAAiB,GAAa,EAAE,CAAC;IAEvC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,IAAI,GAAgC,KAAK,CAAC,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC3E,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,qGAAqG;YACrG,+DAA+D;YAC/D,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;YACpF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,IAAI,EAAE,CAAC;oBACT,IAAI,GAAG,IAAI,CAAC;oBACZ,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC;gBACvC,IAAI;gBACJ,QAAQ;gBACR,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,aAAa;gBACb,OAAO;gBACP,KAAK,EAAE,OAAO,CAAC,KAAK;aACrB,CAAC,CAAC;YACH,IAAI,MAAM,CAAC,UAAU;gBAAE,eAAe,EAAE,CAAC;iBACpC,IAAI,MAAM,CAAC,OAAO;gBAAE,YAAY,EAAE,CAAC;YACxC,IAAI,IAAI,CAAC,UAAU;gBAAE,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC9D,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC7B,SAAS;QACX,CAAC;QAED,kGAAkG;QAClG,uGAAuG;QACvG,2FAA2F;QAC3F,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QACpF,MAAM,SAAS,GACb,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAClF,IAAI,SAAS,EAAE,CAAC;YACd,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,OAAO;QACL,mBAAmB;QACnB,UAAU,EAAE,eAAe;QAC3B,OAAO,EAAE,YAAY;QACrB,mBAAmB;QACnB,iBAAiB;KAClB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type PullMediaOptions, type PullMediaResult } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Pulls media metadata and files from EmDash API into media/ directory with cursor pagination.
|
|
4
|
+
* media/<markdownRef>.yaml
|
|
5
|
+
* media/<markdownRef>.<ext>
|
|
6
|
+
*
|
|
7
|
+
* `markdownRef` is the sanitized storageKey basename (`storageKeyToMarkdownRef`), never the record id, so the
|
|
8
|
+
* mirror on disk shares one stem with the body URL that references it; the original `filename` survives only
|
|
9
|
+
* inside the companion YAML. See `mediaMirrorNames()` for the full naming and collision rule.
|
|
10
|
+
*/
|
|
11
|
+
export declare function pullMedia(options: PullMediaOptions): Promise<PullMediaResult>;
|
|
12
|
+
//# sourceMappingURL=pull_library.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pull_library.d.ts","sourceRoot":"","sources":["../../../src/sync/media/pull_library.ts"],"names":[],"mappings":"AAYA,OAAO,EAGL,KAAK,gBAAgB,EACrB,KAAK,eAAe,EAErB,MAAM,YAAY,CAAC;AAEpB;;;;;;;;GAQG;AACH,wBAAsB,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CA4PnF"}
|