emdash-emit 0.1.13 → 0.1.15

package/dist/sync/media/pull_article.js

@@ -0,0 +1,273 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { unwrapEnvelope } from '../../client/api.js';
+import { ApiError } from '../../client/error.js';
+import { createAuthedFetch } from '../../client/http.js';
+import { extractMediaUlid } from '../../client/mapper.js';
+import { parseMarkdown } from '../../utils/frontmatter.js';
+import { collectArticleReferencedMediaIds, collectArticleStorageKeys } from './body_scan.js';
+import { collectLocalMediaMirrorCandidates } from './companions.js';
+import { syncSingleMediaItem } from './mirror_item.js';
+/**
+ * Pulls media referenced by a specific article (by ULID or file path).
+ */
+export async function pullArticleMedia(options) {
+    const fetchFn = createAuthedFetch(options);
+    const collection = options.collection || 'posts';
+    const downloadFiles = options.downloadFiles !== false;
+    const mediaDir = path.join(options.workspaceRoot, 'media');
+    fs.mkdirSync(mediaDir, { recursive: true });
+    let articleId = options.article;
+    let articleContent = null;
+    // 1. Try resolving as direct file path
+    const directPath = path.isAbsolute(options.article)
+        ? options.article
+        : path.resolve(options.workspaceRoot, options.article);
+    if (fs.existsSync(directPath) && fs.statSync(directPath).isFile()) {
+        articleContent = fs.readFileSync(directPath, 'utf8');
+        try {
+            const parsed = parseMarkdown(articleContent);
+            articleId = parsed.frontmatter.id || articleId;
+        }
+        catch {
+            // Keep articleId as fallback
+        }
+    }
+    else {
+        // 2. Try finding file in draft/published
+        for (const status of ['draft', 'published']) {
+            const statusDir = path.join(options.workspaceRoot, collection, status);
+            if (!fs.existsSync(statusDir))
+                continue;
+            const found = fs
+                .readdirSync(statusDir)
+                .find((f) => f.endsWith('.md') && f.includes(options.article));
+            if (found) {
+                articleContent = fs.readFileSync(path.join(statusDir, found), 'utf8');
+                try {
+                    const parsed = parseMarkdown(articleContent);
+                    articleId = parsed.frontmatter.id || articleId;
+                }
+                catch {
+                    // fallback
+                }
+                break;
+            }
+        }
+    }
+    // 3. If still absent locally, fetch remote article content from EmDash API
+    if (!articleContent) {
+        const url = new URL(`/_emdash/api/content/${collection}/${options.article}`, options.origin);
+        const res = await fetchFn(url.toString(), {
+            headers: {
+                Authorization: `Bearer ${options.token}`,
+                Accept: 'application/json',
+            },
+        });
+        if (!res.ok) {
+            throw new ApiError({
+                message: `Failed to fetch remote article for media resolution: ${res.statusText}`,
+                code: 'ARTICLE_NOT_FOUND',
+                status: res.status,
+                endpoint: url.toString(),
+                suggestion: `Verify that article '${options.article}' exists in collection '${collection}' on remote.`,
+            });
+        }
+        const json = await res.json();
+        const data = unwrapEnvelope(json);
+        const item = (data.item || data);
+        articleId = item.id || options.article;
+        // Build a content string from the remote representation so references can be collected even when
+        // no local file exists yet.
+        const remoteFi = item.featured_image || item.featuredImage;
+        const fiId = extractMediaUlid(remoteFi);
+        const fiPart = fiId ? `featuredImage: ${fiId}\n` : '';
+        const bodyPart = typeof item.content === 'string'
+            ? item.content
+            : JSON.stringify(item.content || item.body || '');
+        articleContent = `---\n${fiPart}---\n${bodyPart}`;
+    }
+    const mediaIds = collectArticleReferencedMediaIds(articleContent);
+    // Body storageKeys are the only handle available when a migrated body bakes the R2 `storageKey`
+    // instead of the D1 record id. They must never be fed to `GET /_emdash/api/media/{id}` as if they
+    // were ids: that lookup keys on the primary key, so it returns 404 for a record that does exist.
+    const storageKeyDescriptors = collectArticleStorageKeys(articleContent);
+    const descriptorsByKey = new Map();
+    for (const descriptor of storageKeyDescriptors) {
+        descriptorsByKey.set(descriptor.storageKey, descriptor);
+    }
+    const allKeys = Array.from(descriptorsByKey.keys()).sort();
+    if (mediaIds.length === 0 && allKeys.length === 0) {
+        return {
+            total: 0,
+            downloaded: 0,
+            skipped: 0,
+            failed: 0,
+            articleId,
+            mediaIds: [],
+            orphanStorageKeys: [],
+        };
+    }
+    let downloadedCount = 0;
+    let skippedCount = 0;
+    // A body ULID that no canonical lookup resolves is the normal shape of a migrated corpus: the URL
+    // carries an R2 storageKey, not a media id. Only a reference that also survives every storageKey
+    // check is a problem, so a failed id lookup alone is never reported as `failed`.
+    let unresolvedIdCount = 0;
+    const recoveredStorageKeys = new Set();
+    // Ids whose canonical lookup succeeded. A body key sharing one of these ULIDs is a confirmed match:
+    // the record exists, it simply carries a storageKey that differs from the one baked into the body.
+    const resolvedRecordIds = new Set();
+    // 1. Canonical path: resolve each referenced media record id and mirror it normally.
+    for (const mId of mediaIds) {
+        try {
+            const mediaUrl = new URL(`/_emdash/api/media/${mId}`, options.origin);
+            const res = await fetchFn(mediaUrl.toString(), {
+                headers: {
+                    Authorization: `Bearer ${options.token}`,
+                    Accept: 'application/json',
+                },
+            });
+            if (!res.ok) {
+                unresolvedIdCount++;
+                continue;
+            }
+            const json = await res.json();
+            const envelope = unwrapEnvelope(json);
+            const item = (envelope.item || envelope);
+            const result = await syncSingleMediaItem({
+                item,
+                mediaDir,
+                origin: options.origin,
+                token: options.token,
+                downloadFiles,
+                fetchFn,
+                cache: options.cache,
+            });
+            if (result.downloaded)
+                downloadedCount++;
+            else if (result.skipped)
+                skippedCount++;
+            resolvedRecordIds.add(mId.toUpperCase());
+            if (item.storageKey)
+                recoveredStorageKeys.add(item.storageKey);
+        }
+        catch {
+            unresolvedIdCount++;
+        }
+    }
+    // 2. Unresolved storageKeys: a body key that no canonical id lookup mirrored is NOT proof of a
+    //    missing record. The record can exist under a different primary key, so Emit re-checks the key
+    //    against confirmed evidence only — a record it just resolved under the same ULID, or the local
+    //    mirror metadata (companion YAML `id` + `storageKey`) — and reports what still fails as
+    //    unresolved. Emit never fetches such an asset and never registers a record for it.
+    const mirrorCandidates = collectLocalMediaMirrorCandidates(mediaDir);
+    const verifiedStorageKeys = [];
+    const orphanStorageKeys = [];
+    for (const key of allKeys) {
+        if (recoveredStorageKeys.has(key))
+            continue;
+        const ulids = (key.match(/[0-9A-HJ-NP-Z]{26}/gi) ?? []).map((u) => u.toUpperCase());
+        // Only two things count as evidence: a record Emit successfully fetched under this exact ULID, or
+        // confirmed local mirror metadata naming it. A body reference alone is never self-confirming — that
+        // is precisely the case where the asset must be reported instead of assumed healthy.
+        const confirmed = ulids.some((ulid) => resolvedRecordIds.has(ulid) || mirrorCandidates.has(ulid));
+        if (confirmed) {
+            verifiedStorageKeys.push(key);
+            continue;
+        }
+        orphanStorageKeys.push(key);
+    }
+    return {
+        total: mediaIds.length + verifiedStorageKeys.length + orphanStorageKeys.length,
+        downloaded: downloadedCount,
+        skipped: skippedCount,
+        failed: orphanStorageKeys.length,
+        unresolvedIds: unresolvedIdCount,
+        articleId,
+        mediaIds,
+        verifiedStorageKeys,
+        orphanStorageKeys,
+    };
+}
+/**
+ * Mirrors only the assets a batch of pulled articles actually references, reusing the records the lightweight
+ * index walk already fetched instead of paying for a per-asset `GET /_emdash/api/media/{id}`.
+ *
+ * This is the mirror half of the default-ON local-first pull: the walk phase builds `RemoteStorageKeyIndex`
+ * (JSON only, no binaries), and this phase downloads the binary + companion YAML for exactly the storageKeys
+ * found in article bodies. A key the walk did not report is NOT proof of a missing record — the record can
+ * exist under a different primary key while the body carries an R2 storageKey — so it is re-checked against
+ * confirmed evidence only (a walked record resolved under that exact ULID, or local mirror metadata) and what
+ * still fails is reported as `orphanStorageKeys`. Emit never fetches such an asset and never registers one.
+ */
+export async function mirrorReferencedAssetsFromIndex(referencedStorageKeys, index, options) {
+    const fetchFn = createAuthedFetch(options);
+    const downloadFiles = options.downloadFiles !== false;
+    const mediaDir = path.join(options.workspaceRoot, 'media');
+    fs.mkdirSync(mediaDir, { recursive: true });
+    const mirroredStorageKeys = new Set();
+    let downloadedCount = 0;
+    let skippedCount = 0;
+    // Walk every referenced key once. A key resolves to a walked record either directly (the walk reported
+    // that exact storageKey) or through a ULID the walk resolved under that record id — the migrated-corpus
+    // case where the body URL bakes an R2 storageKey that differs from the record's own primary key.
+    const referenced = [...referencedStorageKeys];
+    const mirrorCandidates = collectLocalMediaMirrorCandidates(mediaDir);
+    const verifiedStorageKeys = [];
+    const orphanStorageKeys = [];
+    for (const key of referenced) {
+        let item = index.recordsByStorageKey.get(key);
+        if (!item) {
+            // The body key may itself be a record id the walk stored under `recordsById`, or share a ULID with a
+            // walked record. Reuse that record rather than re-fetching it.
+            const ulids = (key.match(/[0-9A-HJ-NP-Z]{26}/gi) ?? []).map((u) => u.toUpperCase());
+            for (const ulid of ulids) {
+                const byId = index.recordsById.get(ulid);
+                if (byId) {
+                    item = byId;
+                    break;
+                }
+            }
+        }
+        if (item) {
+            const result = await syncSingleMediaItem({
+                item,
+                mediaDir,
+                origin: options.origin,
+                token: options.token,
+                downloadFiles,
+                fetchFn,
+                cache: options.cache,
+            });
+            if (result.downloaded)
+                downloadedCount++;
+            else if (result.skipped)
+                skippedCount++;
+            if (item.storageKey)
+                mirroredStorageKeys.add(item.storageKey);
+            mirroredStorageKeys.add(key);
+            continue;
+        }
+        // No walked record owns this key. Only two things count as evidence it is healthy: a local mirror
+        // companion already names the ULID, or the walk stored a matching storageKey. A bare body reference is
+        // never self-confirming — that is precisely the case to report rather than assume healthy.
+        const ulids = (key.match(/[0-9A-HJ-NP-Z]{26}/gi) ?? []).map((u) => u.toUpperCase());
+        const confirmed = index.storageKeys.has(key) ||
+            ulids.some((ulid) => index.recordsById.has(ulid) || mirrorCandidates.has(ulid));
+        if (confirmed) {
+            verifiedStorageKeys.push(key);
+        }
+        else {
+            orphanStorageKeys.push(key);
+        }
+    }
+    return {
+        mirroredStorageKeys,
+        downloaded: downloadedCount,
+        skipped: skippedCount,
+        verifiedStorageKeys,
+        orphanStorageKeys,
+    };
+}
+//# sourceMappingURL=pull_article.js.map

package/dist/sync/media/pull_article.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pull_article.js","sourceRoot":"","sources":["../../../src/sync/media/pull_article.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE1D,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,gCAAgC,EAAE,yBAAyB,EAAE,MAAM,gBAAgB,CAAC;AAC7F,OAAO,EAAE,iCAAiC,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AASvD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAUtC;IACC,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC;IACjD,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,KAAK,KAAK,CAAC;IACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAC3D,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5C,IAAI,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAChC,IAAI,cAAc,GAAkB,IAAI,CAAC;IAEzC,uCAAuC;IACvC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC;QACjD,CAAC,CAAC,OAAO,CAAC,OAAO;QACjB,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEzD,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;QAClE,cAAc,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,aAAa,CAAC,cAAc,CAAC,CAAC;YAC7C,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,SAAS,CAAC;QACjD,CAAC;QAAC,MAAM,CAAC;YACP,6BAA6B;QAC/B,CAAC;IACH,CAAC;SAAM,CAAC;QACN,yCAAyC;QACzC,KAAK,MAAM,MAAM,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,EAAE,CAAC;YAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;YACvE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;gBAAE,SAAS;YACxC,MAAM,KAAK,GAAG,EAAE;iBACb,WAAW,CAAC,SAAS,CAAC;iBACtB,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;YACjE,IAAI,KAAK,EAAE,CAAC;gBACV,cAAc,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC;gBACtE,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,aAAa,CAAC,cAAc,CAAC,CAAC;oBAC7C,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,SAAS,CAAC;gBACjD,CAAC;gBAAC,MAAM,CAAC;oBACP,WAAW;gBACb,CAAC;gBACD,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,wBAAwB,UAAU,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC7F,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YACxC,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,OAAO,CAAC,KAAK,EAAE;gBACxC,MAAM,EAAE,kBAAkB;aAC3B;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,QAAQ,CAAC;gBACjB,OAAO,EAAE,wDAAwD,GAAG,CAAC,UAAU,EAAE;gBACjF,IAAI,EAAE,mBAAmB;gBACzB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE;gBACxB,UAAU,EAAE,wBAAwB,OAAO,CAAC,OAAO,2BAA2B,UAAU,cAAc;aACvG,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,cAAc,CAA0B,IAAI,CAAC,CAAC;QAC3D,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAA4B,CAAC;QAC5D,SAAS,GAAI,IAAI,CAAC,EAAa,IAAI,OAAO,CAAC,OAAO,CAAC;QAEnD,iGAAiG;QACjG,4BAA4B;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,aAAa,CAAC;QAC3D,MAAM,IAAI,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,kBAAkB,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,MAAM,QAAQ,GACZ,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;YAC9B,CAAC,CAAC,IAAI,CAAC,OAAO;YACd,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QACtD,cAAc,GAAG,QAAQ,MAAM,QAAQ,QAAQ,EAAE,CAAC;IACpD,CAAC;IAED,MAAM,QAAQ,GAAG,gCAAgC,CAAC,cAAc,CAAC,CAAC;IAClE,gGAAgG;IAChG,kGAAkG;IAClG,iGAAiG;IACjG,MAAM,qBAAqB,GAAG,yBAAyB,CAAC,cAAc,CAAC,CAAC;IACxE,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAiC,CAAC;IAClE,KAAK,MAAM,UAAU,IAAI,qBAAqB,EAAE,CAAC;QAC/C,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAC1D,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAE3D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClD,OAAO;YACL,KAAK,EAAE,CAAC;YACR,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,CAAC;YACT,SAAS;YACT,QAAQ,EAAE,EAAE;YACZ,iBAAiB,EAAE,EAAE;SACtB,CAAC;IACJ,CAAC;IAED,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,kGAAkG;IAClG,iGAAiG;IACjG,iFAAiF;IACjF,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/C,oGAAoG;IACpG,mGAAmG;IACnG,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE5C,qFAAqF;IACrF,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,sBAAsB,GAAG,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YACtE,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE;gBAC7C,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,OAAO,CAAC,KAAK,EAAE;oBACxC,MAAM,EAAE,kBAAkB;iBAC3B;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,iBAAiB,EAAE,CAAC;gBACpB,SAAS;YACX,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAC9B,MAAM,QAAQ,GAAG,cAAc,CAA0B,IAAI,CAAC,CAAC;YAC/D,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAoB,CAAC;YAE5D,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC;gBACvC,IAAI;gBACJ,QAAQ;gBACR,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,aAAa;gBACb,OAAO;gBACP,KAAK,EAAE,OAAO,CAAC,KAAK;aACrB,CAAC,CAAC;YAEH,IAAI,MAAM,CAAC,UAAU;gBAAE,eAAe,EAAE,CAAC;iBACpC,IAAI,MAAM,CAAC,OAAO;gBAAE,YAAY,EAAE,CAAC;YACxC,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;YACzC,IAAI,IAAI,CAAC,UAAU;gBAAE,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACjE,CAAC;QAAC,MAAM,CAAC;YACP,iBAAiB,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;IAED,+FAA+F;IAC/F,mGAAmG;IACnG,mGAAmG;IACnG,4FAA4F;IAC5F,uFAAuF;IACvF,MAAM,gBAAgB,GAAG,iCAAiC,CAAC,QAAQ,CAAC,CAAC;IACrE,MAAM,mBAAmB,GAAa,EAAE,CAAC;IACzC,MAAM,iBAAiB,GAAa,EAAE,CAAC;IACvC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5C,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QACpF,kGAAkG;QAClG,oGAAoG;QACpG,qFAAqF;QACrF,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAC1B,CAAC,IAAI,EAAE,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,CACpE,CAAC;QACF,IAAI,SAAS,EAAE,CAAC;YACd,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9B,CAAC;IAED,OAAO;QACL,KAAK,EAAE,QAAQ,CAAC,MAAM,GAAG,mBAAmB,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM;QAC9E,UAAU,EAAE,eAAe;QAC3B,OAAO,EAAE,YAAY;QACrB,MAAM,EAAE,iBAAiB,CAAC,MAAM;QAChC,aAAa,EAAE,iBAAiB;QAChC,SAAS;QACT,QAAQ;QACR,mBAAmB;QACnB,iBAAiB;KAClB,CAAC;AACJ,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACnD,qBAAuC,EACvC,KAA4B,EAC5B,OAQC;IAED,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,KAAK,KAAK,CAAC;IACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAC3D,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5C,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9C,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,uGAAuG;IACvG,wGAAwG;IACxG,iGAAiG;IACjG,MAAM,UAAU,GAAG,CAAC,GAAG,qBAAqB,CAAC,CAAC;IAC9C,MAAM,gBAAgB,GAAG,iCAAiC,CAAC,QAAQ,CAAC,CAAC;IACrE,MAAM,mBAAmB,GAAa,EAAE,CAAC;IACzC,MAAM,iBAAiB,GAAa,EAAE,CAAC;IAEvC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,IAAI,GAAgC,KAAK,CAAC,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC3E,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,qGAAqG;YACrG,+DAA+D;YAC/D,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;YACpF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,IAAI,EAAE,CAAC;oBACT,IAAI,GAAG,IAAI,CAAC;oBACZ,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC;gBACvC,IAAI;gBACJ,QAAQ;gBACR,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,aAAa;gBACb,OAAO;gBACP,KAAK,EAAE,OAAO,CAAC,KAAK;aACrB,CAAC,CAAC;YACH,IAAI,MAAM,CAAC,UAAU;gBAAE,eAAe,EAAE,CAAC;iBACpC,IAAI,MAAM,CAAC,OAAO;gBAAE,YAAY,EAAE,CAAC;YACxC,IAAI,IAAI,CAAC,UAAU;gBAAE,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC9D,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC7B,SAAS;QACX,CAAC;QAED,kGAAkG;QAClG,uGAAuG;QACvG,2FAA2F;QAC3F,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QACpF,MAAM,SAAS,GACb,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAClF,IAAI,SAAS,EAAE,CAAC;YACd,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,OAAO;QACL,mBAAmB;QACnB,UAAU,EAAE,eAAe;QAC3B,OAAO,EAAE,YAAY;QACrB,mBAAmB;QACnB,iBAAiB;KAClB,CAAC;AACJ,CAAC"}

package/dist/sync/media/pull_library.d.ts

@@ -0,0 +1,12 @@
+import { type PullMediaOptions, type PullMediaResult } from './types.js';
+/**
+ * Pulls media metadata and files from EmDash API into media/ directory with cursor pagination.
+ * media/<markdownRef>.yaml
+ * media/<markdownRef>.<ext>
+ *
+ * `markdownRef` is the sanitized storageKey basename (`storageKeyToMarkdownRef`), never the record id, so the
+ * mirror on disk shares one stem with the body URL that references it; the original `filename` survives only
+ * inside the companion YAML. See `mediaMirrorNames()` for the full naming and collision rule.
+ */
+export declare function pullMedia(options: PullMediaOptions): Promise<PullMediaResult>;
+//# sourceMappingURL=pull_library.d.ts.map

package/dist/sync/media/pull_library.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pull_library.d.ts","sourceRoot":"","sources":["../../../src/sync/media/pull_library.ts"],"names":[],"mappings":"AAYA,OAAO,EAGL,KAAK,gBAAgB,EACrB,KAAK,eAAe,EAErB,MAAM,YAAY,CAAC;AAEpB;;;;;;;;GAQG;AACH,wBAAsB,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CAuMnF"}

package/dist/sync/media/pull_library.js

@@ -0,0 +1,207 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { unwrapEnvelope } from '../../client/api.js';
+import { ApiError } from '../../client/error.js';
+import { createAuthedFetch } from '../../client/http.js';
+import { DEFAULT_CONFIG } from '../../utils/config.js';
+import { collectArticleStorageKeys } from './body_scan.js';
+import { collectLocalMediaMirrorCandidates } from './companions.js';
+import { syncSingleMediaItem } from './mirror_item.js';
+import { mediaMirrorNames, mediaRecordIdForStem } from './naming_mirror.js';
+import { pullArticleMedia } from './pull_article.js';
+import { findUnmirroredStorageKeys } from './remote_index.js';
+import { MEDIA_LIST_PAGE_SIZE, } from './types.js';
+/**
+ * Pulls media metadata and files from EmDash API into media/ directory with cursor pagination.
+ * media/<markdownRef>.yaml
+ * media/<markdownRef>.<ext>
+ *
+ * `markdownRef` is the sanitized storageKey basename (`storageKeyToMarkdownRef`), never the record id, so the
+ * mirror on disk shares one stem with the body URL that references it; the original `filename` survives only
+ * inside the companion YAML. See `mediaMirrorNames()` for the full naming and collision rule.
+ */
+export async function pullMedia(options) {
+    if (options.article) {
+        return pullArticleMedia({
+            workspaceRoot: options.workspaceRoot,
+            origin: options.origin,
+            token: options.token,
+            refreshToken: options.refreshToken,
+            article: options.article,
+            collection: options.collection,
+            downloadFiles: options.downloadFiles,
+            fetchFn: options.fetchFn,
+            cache: options.cache,
+        });
+    }
+    const fetchFn = createAuthedFetch(options);
+    const targetTotal = options.all ? Infinity : (options.limit ?? DEFAULT_CONFIG.mediaSyncLimit);
+    const downloadFiles = options.downloadFiles !== false;
+    const batchSize = MEDIA_LIST_PAGE_SIZE;
+    const mediaDir = path.join(options.workspaceRoot, 'media');
+    fs.mkdirSync(mediaDir, { recursive: true });
+    // Every mirrored companion is a confirmed storageKey source, including the ones written before Emit
+    // persisted the field, whose `url:` still names the key. Seeding from disk means `--prune` never
+    // deletes an asset that this run's media window simply did not reach.
+    const activeStorageKeys = new Set();
+    for (const candidate of collectLocalMediaMirrorCandidates(mediaDir).values()) {
+        if (candidate.storageKey)
+            activeStorageKeys.add(candidate.storageKey);
+    }
+    // Metadata for the coverage report and the storageKey census. `pullMedia` already walks the library
+    // cursor-by-cursor, so it owns the index instead of paying for a second one.
+    const remoteIndex = {
+        storageKeys: activeStorageKeys,
+        mediaIdsByStorageKey: new Map(),
+        recordsByStorageKey: new Map(),
+        recordsById: new Map(),
+        totalCount: null,
+        complete: false,
+        scannedCount: 0,
+    };
+    const activeMediaBaseNames = new Set();
+    // Only records the remote walk actually reports may authorise a mirror to stay. Seeding this from local
+    // companions instead would make every existing file self-exempt and prune nothing at all.
+    const activeMediaIds = new Set();
+    let totalProcessed = 0;
+    let downloadedCount = 0;
+    let skippedCount = 0;
+    let cursor = undefined;
+    while (totalProcessed < targetTotal) {
+        const remaining = targetTotal - totalProcessed;
+        const currentBatch = Math.min(remaining, batchSize);
+        const url = new URL(`/_emdash/api/media`, options.origin);
+        url.searchParams.set('limit', String(currentBatch));
+        if (cursor) {
+            url.searchParams.set('cursor', cursor);
+        }
+        const res = await fetchFn(url.toString(), {
+            method: 'GET',
+            headers: {
+                Authorization: `Bearer ${options.token}`,
+                Accept: 'application/json',
+            },
+        });
+        if (!res.ok) {
+            throw new ApiError({
+                message: `Failed to list media: ${res.statusText}`,
+                code: 'MEDIA_LIST_FAILED',
+                status: res.status,
+                endpoint: url.toString(),
+                suggestion: 'Verify network connection and token permissions.',
+            });
+        }
+        const json = await res.json();
+        const data = unwrapEnvelope(json);
+        if (typeof data.totalCount === 'number')
+            remoteIndex.totalCount = data.totalCount;
+        const rawItems = (Array.isArray(data.items) ? data.items : Array.isArray(data) ? data : []);
+        if (rawItems.length === 0) {
+            // An empty page ends the listing regardless of any cursor the server echoes back.
+            remoteIndex.complete = true;
+            break;
+        }
+        for (const item of rawItems) {
+            const names = mediaMirrorNames(item);
+            activeMediaBaseNames.add(names.markdownRef);
+            for (const legacy of names.legacyBaseNames) {
+                activeMediaBaseNames.add(legacy);
+            }
+            activeMediaIds.add(item.id);
+            if (item.storageKey) {
+                activeStorageKeys.add(item.storageKey);
+                remoteIndex.mediaIdsByStorageKey.set(item.storageKey, item.id);
+            }
+            const result = await syncSingleMediaItem({
+                item,
+                mediaDir,
+                origin: options.origin,
+                token: options.token,
+                downloadFiles,
+                fetchFn,
+                cache: options.cache,
+            });
+            if (result.downloaded)
+                downloadedCount++;
+            else if (result.skipped)
+                skippedCount++;
+            totalProcessed++;
+            remoteIndex.scannedCount++;
+            if (totalProcessed >= targetTotal) {
+                break;
+            }
+        }
+        if (!data.nextCursor || typeof data.nextCursor !== 'string') {
+            remoteIndex.complete = true;
+            break;
+        }
+        cursor = data.nextCursor;
+    }
+    // Pruning logic for media: if --prune or --all is enabled, delete local media not present on remote
+    let deletedCount = 0;
+    if ((options.prune || options.all) && fs.existsSync(mediaDir)) {
+        const files = fs.readdirSync(mediaDir);
+        for (const file of files) {
+            // Leftover `legacy_*` mirrors from the removed file-route fallback are never matched by the
+            // mediaId pattern below, so they survive pruning and must be deleted by hand.
+            if (file.startsWith('legacy_'))
+                continue;
+            // A mirror is identified with its companion YAML: the stem names the storageKey, and the record id
+            // lives inside the companion. Only a file Emit can attribute to a record may be pruned — an
+            // unattributable migrated name is kept, because deleting an asset over a guess is unrecoverable.
+            const stem = file.replace(/\.[^.]+$/, '');
+            if (activeMediaBaseNames.has(stem) || activeMediaBaseNames.has(stem.replace(/^\d{8}_/, ''))) {
+                continue;
+            }
+            const companionId = mediaRecordIdForStem(mediaDir, stem);
+            if (companionId && !activeMediaIds.has(companionId)) {
+                try {
+                    fs.unlinkSync(path.join(mediaDir, file));
+                    deletedCount++;
+                }
+                catch {
+                    // Ignore unlink errors
+                }
+            }
+        }
+    }
+    // A library-wide walk is exhaustive, so it doubles as the storageKey census: every article reference
+    // that neither a local companion nor any remote record names is reported. The list endpoint exposes no
+    // storageKey filter, which is why the census requires the full walk `--all` already performs.
+    let referencedUnmirroredStorageKeys = undefined;
+    if (options.all) {
+        const referenced = new Set();
+        for (const status of ['draft', 'published']) {
+            const statusDir = path.join(options.workspaceRoot, options.collection || 'posts', status);
+            if (!fs.existsSync(statusDir))
+                continue;
+            for (const file of fs.readdirSync(statusDir)) {
+                if (!file.endsWith('.md'))
+                    continue;
+                try {
+                    const content = fs.readFileSync(path.join(statusDir, file), 'utf8');
+                    for (const descriptor of collectArticleStorageKeys(content)) {
+                        referenced.add(descriptor.storageKey);
+                    }
+                }
+                catch {
+                    // An unreadable article contributes nothing to the census.
+                }
+            }
+        }
+        if (referenced.size > 0) {
+            referencedUnmirroredStorageKeys = findUnmirroredStorageKeys(referenced, activeStorageKeys, remoteIndex);
+        }
+    }
+    return {
+        total: totalProcessed,
+        downloaded: downloadedCount,
+        skipped: skippedCount,
+        deleted: deletedCount,
+        remoteTotalCount: remoteIndex.complete
+            ? (remoteIndex.totalCount ?? remoteIndex.scannedCount)
+            : remoteIndex.totalCount,
+        referencedUnmirroredStorageKeys,
+    };
+}
+//# sourceMappingURL=pull_library.js.map

package/dist/sync/media/pull_library.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pull_library.js","sourceRoot":"","sources":["../../../src/sync/media/pull_library.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,gBAAgB,CAAC;AAC3D,OAAO,EAAE,iCAAiC,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,yBAAyB,EAAE,MAAM,mBAAmB,CAAC;AAC9D,OAAO,EACL,oBAAoB,GAKrB,MAAM,YAAY,CAAC;AAEpB;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB;IACvD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,gBAAgB,CAAC;YACtB,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,IAAI,cAAc,CAAC,cAAc,CAAC,CAAC;IAC9F,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,KAAK,KAAK,CAAC;IACtD,MAAM,SAAS,GAAG,oBAAoB,CAAC;IAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAC3D,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,oGAAoG;IACpG,iGAAiG;IACjG,sEAAsE;IACtE,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC5C,KAAK,MAAM,SAAS,IAAI,iCAAiC,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;QAC7E,IAAI,SAAS,CAAC,UAAU;YAAE,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IACxE,CAAC;IAED,oGAAoG;IACpG,6EAA6E;IAC7E,MAAM,WAAW,GAA0B;QACzC,WAAW,EAAE,iBAAiB;QAC9B,oBAAoB,EAAE,IAAI,GAAG,EAAkB;QAC/C,mBAAmB,EAAE,IAAI,GAAG,EAA2B;QACvD,WAAW,EAAE,IAAI,GAAG,EAA2B;QAC/C,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE,KAAK;QACf,YAAY,EAAE,CAAC;KAChB,CAAC;IAEF,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/C,wGAAwG;IACxG,0FAA0F;IAC1F,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IAEzC,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,MAAM,GAAuB,SAAS,CAAC;IAE3C,OAAO,cAAc,GAAG,WAAW,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,WAAW,GAAG,cAAc,CAAC;QAC/C,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAEpD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,oBAAoB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC1D,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;QACpD,IAAI,MAAM,EAAE,CAAC;YACX,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACzC,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YACxC,MAAM,EAAE,KAAK;YACb,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,OAAO,CAAC,KAAK,EAAE;gBACxC,MAAM,EAAE,kBAAkB;aAC3B;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,QAAQ,CAAC;gBACjB,OAAO,EAAE,yBAAyB,GAAG,CAAC,UAAU,EAAE;gBAClD,IAAI,EAAE,mBAAmB;gBACzB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE;gBACxB,UAAU,EAAE,kDAAkD;aAC/D,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,cAAc,CAA0B,IAAI,CAAC,CAAC;QAC3D,IAAI,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ;YAAE,WAAW,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClF,MAAM,QAAQ,GAAG,CACf,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CACpD,CAAC;QAEvB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,kFAAkF;YAClF,WAAW,CAAC,QAAQ,GAAG,IAAI,CAAC;YAC5B,MAAM;QACR,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;YACrC,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAC5C,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;gBAC3C,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACnC,CAAC;YACD,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC5B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACvC,WAAW,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YACjE,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC;gBACvC,IAAI;gBACJ,QAAQ;gBACR,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,aAAa;gBACb,OAAO;gBACP,KAAK,EAAE,OAAO,CAAC,KAAK;aACrB,CAAC,CAAC;YAEH,IAAI,MAAM,CAAC,UAAU;gBAAE,eAAe,EAAE,CAAC;iBACpC,IAAI,MAAM,CAAC,OAAO;gBAAE,YAAY,EAAE,CAAC;YAExC,cAAc,EAAE,CAAC;YACjB,WAAW,CAAC,YAAY,EAAE,CAAC;YAC3B,IAAI,cAAc,IAAI,WAAW,EAAE,CAAC;gBAClC,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;YAC5D,WAAW,CAAC,QAAQ,GAAG,IAAI,CAAC;YAC5B,MAAM;QACR,CAAC;QACD,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,oGAAoG;IACpG,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,4FAA4F;YAC5F,8EAA8E;YAC9E,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;gBAAE,SAAS;YACzC,mGAAmG;YACnG,4FAA4F;YAC5F,iGAAiG;YACjG,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YAC1C,IAAI,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC;gBAC5F,SAAS;YACX,CAAC;YACD,MAAM,WAAW,GAAG,oBAAoB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YACzD,IAAI,WAAW,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;gBACpD,IAAI,CAAC;oBACH,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;oBACzC,YAAY,EAAE,CAAC;gBACjB,CAAC;gBAAC,MAAM,CAAC;oBACP,uBAAuB;gBACzB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,qGAAqG;IACrG,uGAAuG;IACvG,8FAA8F;IAC9F,IAAI,+BAA+B,GAAyB,SAAS,CAAC;IACtE,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;QACrC,KAAK,MAAM,MAAM,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,EAAE,CAAC;YAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,UAAU,IAAI,OAAO,EAAE,MAAM,CAAC,CAAC;YAC1F,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;gBAAE,SAAS;YACxC,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;oBAAE,SAAS;gBACpC,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;oBACpE,KAAK,MAAM,UAAU,IAAI,yBAAyB,CAAC,OAAO,CAAC,EAAE,CAAC;wBAC5D,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;oBACxC,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,2DAA2D;gBAC7D,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,UAAU,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACxB,+BAA+B,GAAG,yBAAyB,CACzD,UAAU,EACV,iBAAiB,EACjB,WAAW,CACZ,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK,EAAE,cAAc;QACrB,UAAU,EAAE,eAAe;QAC3B,OAAO,EAAE,YAAY;QACrB,OAAO,EAAE,YAAY;QACrB,gBAAgB,EAAE,WAAW,CAAC,QAAQ;YACpC,CAAC,CAAC,CAAC,WAAW,CAAC,UAAU,IAAI,WAAW,CAAC,YAAY,CAAC;YACtD,CAAC,CAAC,WAAW,CAAC,UAAU;QAC1B,+BAA+B;KAChC,CAAC;AACJ,CAAC"}

package/dist/sync/media/remote_index.d.ts

@@ -0,0 +1,18 @@
+import { type RemoteStorageKeyIndex } from './types.js';
+export declare function collectRemoteStorageKeys(options: {
+    workspaceRoot: string;
+    origin: string;
+    token: string;
+    refreshToken?: string;
+    /** Stop after this many records. Defaults to unlimited, which is the whole library. */
+    maxItems?: number;
+    fetchFn?: typeof fetch;
+}): Promise<RemoteStorageKeyIndex>;
+/**
+ * Reports body storageKeys that no local mirror names and that a full remote library walk does not
+ * contain. A key is only listed when the walk was complete: an early stop proves nothing about the
+ * rest of the library, so an incomplete walk yields nothing. Even a confirmed absence is reported as a
+ * lookup result rather than as a missing record — Emit mirrors records and never repairs server data.
+ */
+export declare function findUnmirroredStorageKeys(referenced: Iterable<string>, localMirrorKeys: Iterable<string>, remoteIndex: RemoteStorageKeyIndex): string[];
+//# sourceMappingURL=remote_index.d.ts.map

package/dist/sync/media/remote_index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote_index.d.ts","sourceRoot":"","sources":["../../../src/sync/media/remote_index.ts"],"names":[],"mappings":"AAGA,OAAO,EAA8C,KAAK,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAEpG,wBAAsB,wBAAwB,CAAC,OAAO,EAAE;IACtD,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uFAAuF;IACvF,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACxB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CA6EjC;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CACvC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,EAC5B,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,EACjC,WAAW,EAAE,qBAAqB,GACjC,MAAM,EAAE,CASV"}

package/dist/sync/media/remote_index.js

@@ -0,0 +1,94 @@
+import { unwrapEnvelope } from '../../client/api.js';
+import { ApiError } from '../../client/error.js';
+import { createAuthedFetch } from '../../client/http.js';
+import { MEDIA_LIST_PAGE_SIZE } from './types.js';
+export async function collectRemoteStorageKeys(options) {
+    const fetchFn = createAuthedFetch(options);
+    const batchSize = MEDIA_LIST_PAGE_SIZE;
+    const maxItems = options.maxItems ?? Infinity;
+    const storageKeys = new Set();
+    const mediaIdsByStorageKey = new Map();
+    const recordsByStorageKey = new Map();
+    const recordsById = new Map();
+    let totalCount = null;
+    let scanned = 0;
+    let complete = false;
+    let cursor = undefined;
+    for (;;) {
+        const remaining = maxItems - scanned;
+        if (remaining <= 0)
+            break;
+        const currentBatch = Math.min(remaining, batchSize);
+        const url = new URL('/_emdash/api/media', options.origin);
+        url.searchParams.set('limit', String(currentBatch));
+        if (cursor)
+            url.searchParams.set('cursor', cursor);
+        const res = await fetchFn(url.toString(), {
+            headers: {
+                Authorization: `Bearer ${options.token}`,
+                Accept: 'application/json',
+            },
+        });
+        if (!res.ok) {
+            throw new ApiError({
+                message: `Failed to list media: ${res.statusText}`,
+                code: 'MEDIA_LIST_FAILED',
+                status: res.status,
+                endpoint: url.toString(),
+                suggestion: 'Verify network connection and token permissions.',
+            });
+        }
+        const data = unwrapEnvelope(await res.json());
+        if (typeof data.totalCount === 'number')
+            totalCount = data.totalCount;
+        const rawItems = (Array.isArray(data.items) ? data.items : Array.isArray(data.data) ? data.data : []);
+        for (const item of rawItems) {
+            scanned++;
+            if (item.id)
+                recordsById.set(item.id.toUpperCase(), item);
+            if (typeof item.storageKey === 'string' && item.storageKey) {
+                storageKeys.add(item.storageKey);
+                // First record wins a repeated key, so reporting a match stays deterministic.
+                if (item.id && !mediaIdsByStorageKey.has(item.storageKey)) {
+                    mediaIdsByStorageKey.set(item.storageKey, item.id);
+                }
+                if (!recordsByStorageKey.has(item.storageKey)) {
+                    recordsByStorageKey.set(item.storageKey, item);
+                }
+            }
+        }
+        if (rawItems.length === 0 || typeof data.nextCursor !== 'string' || !data.nextCursor) {
+            complete = true;
+            break;
+        }
+        cursor = data.nextCursor;
+    }
+    return {
+        storageKeys,
+        mediaIdsByStorageKey,
+        recordsByStorageKey,
+        recordsById,
+        totalCount,
+        complete,
+        scannedCount: scanned,
+    };
+}
+/**
+ * Reports body storageKeys that no local mirror names and that a full remote library walk does not
+ * contain. A key is only listed when the walk was complete: an early stop proves nothing about the
+ * rest of the library, so an incomplete walk yields nothing. Even a confirmed absence is reported as a
+ * lookup result rather than as a missing record — Emit mirrors records and never repairs server data.
+ */
+export function findUnmirroredStorageKeys(referenced, localMirrorKeys, remoteIndex) {
+    if (!remoteIndex.complete)
+        return [];
+    const mirrored = new Set(localMirrorKeys);
+    const unresolved = new Set();
+    for (const key of referenced) {
+        if (mirrored.has(key) || remoteIndex.storageKeys.has(key))
+            continue;
+        unresolved.add(key);
+    }
+    return Array.from(unresolved).sort();
+}
+//# sourceMappingURL=remote_index.js.map

package/dist/sync/media/remote_index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote_index.js","sourceRoot":"","sources":["../../../src/sync/media/remote_index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAoD,MAAM,YAAY,CAAC;AAEpG,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,OAQ9C;IACC,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,oBAAoB,CAAC;IACvC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,QAAQ,CAAC;IAE9C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvD,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC/D,MAAM,WAAW,GAAG,IAAI,GAAG,EAA2B,CAAC;IACvD,IAAI,UAAU,GAAkB,IAAI,CAAC;IACrC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,MAAM,GAAuB,SAAS,CAAC;IAE3C,SAAS,CAAC;QACR,MAAM,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;QACrC,IAAI,SAAS,IAAI,CAAC;YAAE,MAAM;QAC1B,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAEpD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,oBAAoB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC1D,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;QACpD,IAAI,MAAM;YAAE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAEnD,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YACxC,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,OAAO,CAAC,KAAK,EAAE;gBACxC,MAAM,EAAE,kBAAkB;aAC3B;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,QAAQ,CAAC;gBACjB,OAAO,EAAE,yBAAyB,GAAG,CAAC,UAAU,EAAE;gBAClD,IAAI,EAAE,mBAAmB;gBACzB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE;gBACxB,UAAU,EAAE,kDAAkD;aAC/D,CAAC,CAAC;QACL,CAAC;QAED,MAAM,IAAI,GAAG,cAAc,CAA0B,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACvE,IAAI,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ;YAAE,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACtE,MAAM,QAAQ,GAAG,CACf,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAC9D,CAAC;QAEvB,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,OAAO,EAAE,CAAC;YACV,IAAI,IAAI,CAAC,EAAE;gBAAE,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,CAAC;YAC1D,IAAI,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC3D,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACjC,8EAA8E;gBAC9E,IAAI,IAAI,CAAC,EAAE,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC1D,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;gBACrD,CAAC;gBACD,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC9C,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;gBACjD,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACrF,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM;QACR,CAAC;QACD,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,OAAO;QACL,WAAW;QACX,oBAAoB;QACpB,mBAAmB;QACnB,WAAW;QACX,UAAU;QACV,QAAQ;QACR,YAAY,EAAE,OAAO;KACtB,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,yBAAyB,CACvC,UAA4B,EAC5B,eAAiC,EACjC,WAAkC;IAElC,IAAI,CAAC,WAAW,CAAC,QAAQ;QAAE,OAAO,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QACpE,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;AACvC,CAAC"}