@thallylabs/migrate 0.2.28 → 0.2.30

package/dist/url-migration.js

@@ -463,6 +463,73 @@ function isPassiveSvg(content) {
   return isPassive && sawRoot && depth === 0;
 }
 
+// src/url-asset-cache.ts
+function createMigrationAssetCache() {
+  const settled = /* @__PURE__ */ new Map();
+  const pending = /* @__PURE__ */ new Map();
+  const remember = (url, asset) => {
+    settled.delete(url);
+    settled.set(url, asset);
+    if (settled.size > 480) settled.delete(settled.keys().next().value);
+    return asset;
+  };
+  const get = (url) => {
+    const asset = settled.get(url);
+    if (asset) return Promise.resolve(remember(url, asset));
+    return pending.get(url);
+  };
+  return {
+    remember,
+    get,
+    has(url) {
+      return settled.has(url) || pending.has(url);
+    },
+    get size() {
+      return settled.size + pending.size;
+    },
+    load(url, acquire) {
+      const existing = get(url);
+      if (existing) return existing;
+      const request = Promise.resolve().then(acquire).then((asset) => {
+        pending.delete(url);
+        return remember(url, asset);
+      }, (error) => {
+        pending.delete(url);
+        throw error;
+      });
+      pending.set(url, request);
+      return request;
+    },
+    delete(url) {
+      settled.delete(url);
+    }
+  };
+}
+
+// src/url-journal-clone.ts
+function cloneUrlMigrationCheckpoint(checkpoint) {
+  const ancestors = /* @__PURE__ */ new Set();
+  const clone = (value) => {
+    if (value === null || typeof value === "string" || typeof value === "boolean") return value;
+    if (typeof value === "number") return Number.isFinite(value) ? value === 0 ? 0 : value : null;
+    if (typeof value === "undefined" || typeof value === "function" || typeof value === "symbol") return void 0;
+    if (typeof value !== "object") throw new TypeError("Checkpoint must contain JSON-safe values.");
+    if (ancestors.has(value)) throw new TypeError("Checkpoint must not contain circular references.");
+    if (!Array.isArray(value) && Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) throw new TypeError("Checkpoint must contain plain JSON containers.");
+    ancestors.add(value);
+    try {
+      if (Array.isArray(value)) return Array.from(value, (entry) => clone(entry) ?? null);
+      return Object.fromEntries(Object.entries(value).flatMap(([key, entry]) => {
+        const copied = clone(entry);
+        return copied === void 0 ? [] : [[key, copied]];
+      }));
+    } finally {
+      ancestors.delete(value);
+    }
+  };
+  return clone(checkpoint);
+}
+
 // src/url-images.ts
 function migrationImageUrl(value, base) {
   const url = new URL(value, base);
@@ -3191,7 +3258,8 @@ async function runUrlMigration(options) {
   const batchSize = Math.max(1, Math.min(500, Math.floor(options.batchSize ?? 50)));
   if (!Number.isFinite(concurrency) || !Number.isFinite(batchSize) || options.maxCaptures !== void 0 && (!Number.isInteger(options.maxCaptures) || options.maxCaptures < 1) || options.maxTotalBytes !== void 0 && (!Number.isFinite(options.maxTotalBytes) || options.maxTotalBytes < 1)) throw new Error("Migration batch, concurrency and byte limits must be positive finite numbers.");
   const captureVersion = options.captureVersion ?? "fetch-v1";
-  const checkpoint = options.checkpoint ? JSON.parse(JSON.stringify(options.checkpoint)) : { schemaVersion: 1, converterVersion: CONVERTER_VERSION, sourceUrl, scopePath, captureVersion, records: [], navigation: [] };
+  if (options.checkpoint && options.checkpoint.records.length > MAX_RECORDS) throw new Error("Checkpoint inventory exceeds its bound.");
+  const checkpoint = options.checkpoint ? cloneUrlMigrationCheckpoint(options.checkpoint) : { schemaVersion: 1, converterVersion: CONVERTER_VERSION, sourceUrl, scopePath, captureVersion, records: [], navigation: [] };
   if (checkpoint.schemaVersion !== 1 || checkpoint.converterVersion !== CONVERTER_VERSION || checkpoint.sourceUrl !== sourceUrl || checkpoint.scopePath !== scopePath || checkpoint.captureVersion !== captureVersion) throw new Error("Checkpoint does not match this source, scope or capture/converter version.");
   if (checkpoint.records.length > MAX_RECORDS) throw new Error("Checkpoint inventory exceeds its bound.");
   const records = /* @__PURE__ */ new Map();
@@ -3232,6 +3300,33 @@ async function runUrlMigration(options) {
   enqueue(sourceUrl, sourceUrl, "submitted");
   for (const path of ["llms.txt", "sitemap.xml", "robots.txt"]) enqueue(`${scopePath === "/" ? "" : scopePath}/${path}`, sourceUrl, "discovery-index");
   if (scopePath !== "/") for (const path of ["llms.txt", "sitemap.xml", "robots.txt"]) enqueue(`/${path}`, sourceUrl, "discovery-index");
+  const enqueueArticleLinks = (links, base, discoveredBy) => {
+    for (const link of links) {
+      try {
+        const url = new URL(link, base);
+        if (!/\.(?:png|jpe?g|gif|svg|webp|avif|pdf|zip|mp4|webm|css|js)$/i.test(url.pathname)) enqueue(link, base, discoveredBy);
+      } catch {
+      }
+    }
+  };
+  const enqueueNavigation = (nodes, base) => {
+    for (const node of nodes) {
+      if (node.url) enqueue(node.url, base, "source-navigation");
+      enqueueNavigation(node.children ?? [], base);
+    }
+  };
+  for (const record of [...records.values()]) {
+    if (!record.evidence) continue;
+    const final = canonical(record.evidence.finalUrl, sourceUrl);
+    if (!final) continue;
+    if (record.outcome === "imported") enqueueArticleLinks(record.evidence.links, final, record.url);
+    else if (record.outcome === "excluded") {
+      if (classifyMigrationResponse({ finalUrl: new URL(final), contentType: record.evidence.contentType, body: "" }) === "index") {
+        for (const link of record.evidence.links) enqueue(link, final, record.url);
+      }
+    }
+  }
+  enqueueNavigation(checkpoint.navigation, sourceUrl);
   const fetcher = options.fetcher ?? defaultMigrationFetcher;
   const capture = options.captureProvider?.capture.bind(options.captureProvider) ?? (async ({ url }) => fetcher(url, { accept: ACCEPT }));
   const assetFetcher = options.assetFetcher ?? (!options.fetcher && !options.captureProvider ? defaultAssetFetcher : void 0);
@@ -3239,9 +3334,21 @@ async function runUrlMigration(options) {
   const finalUrls = new Map([...records.values()].filter((record) => record.evidence && record.page).map((record) => [canonical(record.evidence.finalUrl, sourceUrl), record.url]));
   let imported = usedIds.size;
   const assetPayloads = new Map(checkpoint.records.flatMap((record) => (record.assets ?? []).map((asset) => [asset.sha256, asset])));
-  const assetRequests = /* @__PURE__ */ new Map();
+  const assetBytes = /* @__PURE__ */ new Map();
+  const retainAsset = (asset) => {
+    if (!acceptedAsset(asset)) throw new Error("Asset MIME/signature is unsupported or unsafe to serve.");
+    const sha256 = hash(asset.content);
+    const content = assetBytes.get(sha256) ?? Buffer.from(asset.content);
+    assetBytes.set(sha256, content);
+    return { ...asset, content };
+  };
+  const assetRequests = createMigrationAssetCache();
   for (const record of checkpoint.records) for (const asset of record.assets ?? []) {
-    if (!assetRequests.has(asset.url)) assetRequests.set(asset.url, Promise.resolve({ url: asset.url, contentType: asset.contentType, content: Buffer.from(asset.base64, "base64") }));
+    if (!assetRequests.has(asset.url)) {
+      const content = assetBytes.get(asset.sha256) ?? Buffer.from(asset.base64, "base64");
+      assetBytes.set(asset.sha256, content);
+      assetRequests.remember(asset.url, { url: asset.url, contentType: asset.contentType, content });
+    }
   }
   let totalBytes = checkpoint.records.reduce((sum, record) => sum + (record.evidence?.byteLength ?? 0), 0) + [...assetPayloads.values()].reduce((sum, asset) => sum + Buffer.byteLength(asset.base64, "base64"), 0);
   let budgetExhausted = false;
@@ -3353,10 +3460,10 @@ ${difference}`;
         try {
           let assetRequest = assetRequests.get(url);
           if (!suppliedAssets.has(url) && !assetRequest && assetFetcher) {
-            assetRequest = assetFetcher(new URL(url), options.signal);
-            assetRequests.set(url, assetRequest);
+            assetRequest = assetRequests.load(url, () => assetFetcher(new URL(url), options.signal).then(retainAsset));
           }
-          const asset = suppliedAssets.get(url) ?? (assetRequest ? await assetRequest : null);
+          const supplied = suppliedAssets.get(url);
+          const asset = supplied ? retainAsset(supplied) : assetRequest ? await assetRequest : null;
           if (!asset) throw new Error("Capture host did not supply asset bytes or a binary fetch boundary.");
           const extension = acceptedAsset(asset);
           if (!extension) throw new Error("Asset MIME/signature is unsupported or unsafe to serve.");
@@ -3373,7 +3480,7 @@ ${difference}`;
           const receipt = { url, path, sha256, contentType: asset.contentType, base64: existing?.base64 ?? Buffer.from(asset.content).toString("base64") };
           record.assets.push(receipt);
           if (!existing) assetPayloads.set(sha256, receipt);
-          if (!assetRequests.has(url)) assetRequests.set(url, Promise.resolve(asset));
+          if (!assetRequests.has(url)) assetRequests.remember(url, asset);
           page.body = rewriteMarkdownDestinations(page.body, (destination) => canonical(destination, final) === url ? `/${path}${new URL(destination, final).hash}` : destination);
         } catch (error) {
           assetRequests.delete(url);
@@ -3406,20 +3513,8 @@ ${difference}`;
         checkpoint.navigation = navigation;
         navigationChanged = true;
       }
-      for (const link of evidence.links) {
-        try {
-          const url = new URL(link, final);
-          if (!/\.(?:png|jpe?g|gif|svg|webp|avif|pdf|zip|mp4|webm|css|js)$/i.test(url.pathname)) enqueue(link, final, record.url);
-        } catch {
-        }
-      }
-      const visit = (nodes) => {
-        for (const node of nodes) {
-          if (node.url) enqueue(node.url, final, "source-navigation");
-          visit(node.children ?? []);
-        }
-      };
-      visit(navigation);
+      enqueueArticleLinks(evidence.links, final, record.url);
+      enqueueNavigation(navigation, final);
     } catch (error) {
       if (options.signal?.aborted) return;
       const wasImported = record.outcome === "imported";
@@ -3489,7 +3584,13 @@ ${difference}`;
   const validation = validateUrlMigration(checkpoint, Boolean(options.signal?.aborted));
   validation.issues.push(...navigationIssues);
   if (navigationIssues.length && validation.status === "passed") validation.status = "partial";
-  const assets = new Map(checkpoint.records.flatMap((record) => (record.assets ?? []).map((asset) => [asset.path, { path: asset.path, content: Buffer.from(asset.base64, "base64") }])));
+  const assets = /* @__PURE__ */ new Map();
+  for (const record of checkpoint.records) for (const asset of record.assets ?? []) {
+    if (!assets.has(asset.path)) {
+      const content = assetBytes.get(asset.sha256) ?? Buffer.from(asset.base64, "base64");
+      assets.set(asset.path, { path: asset.path, content: Buffer.from(content.buffer, content.byteOffset, content.byteLength) });
+    }
+  }
   const bundle = {
     sourceUrl,
     sourceKind: "url",