@thallylabs/migrate 0.2.28 → 0.2.29

package/dist/index.js

@@ -13876,6 +13876,73 @@ function isPassiveSvg(content) {
   return isPassive && sawRoot && depth === 0;
 }
 
+// src/url-asset-cache.ts
+function createMigrationAssetCache() {
+  const settled = /* @__PURE__ */ new Map();
+  const pending = /* @__PURE__ */ new Map();
+  const remember = (url, asset) => {
+    settled.delete(url);
+    settled.set(url, asset);
+    if (settled.size > 480) settled.delete(settled.keys().next().value);
+    return asset;
+  };
+  const get = (url) => {
+    const asset = settled.get(url);
+    if (asset) return Promise.resolve(remember(url, asset));
+    return pending.get(url);
+  };
+  return {
+    remember,
+    get,
+    has(url) {
+      return settled.has(url) || pending.has(url);
+    },
+    get size() {
+      return settled.size + pending.size;
+    },
+    load(url, acquire) {
+      const existing = get(url);
+      if (existing) return existing;
+      const request2 = Promise.resolve().then(acquire).then((asset) => {
+        pending.delete(url);
+        return remember(url, asset);
+      }, (error) => {
+        pending.delete(url);
+        throw error;
+      });
+      pending.set(url, request2);
+      return request2;
+    },
+    delete(url) {
+      settled.delete(url);
+    }
+  };
+}
+
+// src/url-journal-clone.ts
+function cloneUrlMigrationCheckpoint(checkpoint) {
+  const ancestors = /* @__PURE__ */ new Set();
+  const clone = (value) => {
+    if (value === null || typeof value === "string" || typeof value === "boolean") return value;
+    if (typeof value === "number") return Number.isFinite(value) ? value === 0 ? 0 : value : null;
+    if (typeof value === "undefined" || typeof value === "function" || typeof value === "symbol") return void 0;
+    if (typeof value !== "object") throw new TypeError("Checkpoint must contain JSON-safe values.");
+    if (ancestors.has(value)) throw new TypeError("Checkpoint must not contain circular references.");
+    if (!Array.isArray(value) && Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) throw new TypeError("Checkpoint must contain plain JSON containers.");
+    ancestors.add(value);
+    try {
+      if (Array.isArray(value)) return Array.from(value, (entry) => clone(entry) ?? null);
+      return Object.fromEntries(Object.entries(value).flatMap(([key, entry]) => {
+        const copied = clone(entry);
+        return copied === void 0 ? [] : [[key, copied]];
+      }));
+    } finally {
+      ancestors.delete(value);
+    }
+  };
+  return clone(checkpoint);
+}
+
 // src/url-run.ts
 var CONVERTER_VERSION = "url-evidence-v2";
 var parser2 = unified4().use(remarkParse4).use(remarkMdx4);
@@ -14250,7 +14317,8 @@ async function runUrlMigration(options) {
   const batchSize = Math.max(1, Math.min(500, Math.floor(options.batchSize ?? 50)));
   if (!Number.isFinite(concurrency) || !Number.isFinite(batchSize) || options.maxCaptures !== void 0 && (!Number.isInteger(options.maxCaptures) || options.maxCaptures < 1) || options.maxTotalBytes !== void 0 && (!Number.isFinite(options.maxTotalBytes) || options.maxTotalBytes < 1)) throw new Error("Migration batch, concurrency and byte limits must be positive finite numbers.");
   const captureVersion = options.captureVersion ?? "fetch-v1";
-  const checkpoint = options.checkpoint ? JSON.parse(JSON.stringify(options.checkpoint)) : { schemaVersion: 1, converterVersion: CONVERTER_VERSION, sourceUrl, scopePath, captureVersion, records: [], navigation: [] };
+  if (options.checkpoint && options.checkpoint.records.length > MAX_RECORDS) throw new Error("Checkpoint inventory exceeds its bound.");
+  const checkpoint = options.checkpoint ? cloneUrlMigrationCheckpoint(options.checkpoint) : { schemaVersion: 1, converterVersion: CONVERTER_VERSION, sourceUrl, scopePath, captureVersion, records: [], navigation: [] };
   if (checkpoint.schemaVersion !== 1 || checkpoint.converterVersion !== CONVERTER_VERSION || checkpoint.sourceUrl !== sourceUrl || checkpoint.scopePath !== scopePath || checkpoint.captureVersion !== captureVersion) throw new Error("Checkpoint does not match this source, scope or capture/converter version.");
   if (checkpoint.records.length > MAX_RECORDS) throw new Error("Checkpoint inventory exceeds its bound.");
   const records = /* @__PURE__ */ new Map();
@@ -14298,9 +14366,21 @@ async function runUrlMigration(options) {
   const finalUrls = new Map([...records.values()].filter((record) => record.evidence && record.page).map((record) => [canonical(record.evidence.finalUrl, sourceUrl), record.url]));
   let imported = usedIds.size;
   const assetPayloads = new Map(checkpoint.records.flatMap((record) => (record.assets ?? []).map((asset) => [asset.sha256, asset])));
-  const assetRequests = /* @__PURE__ */ new Map();
+  const assetBytes = /* @__PURE__ */ new Map();
+  const retainAsset = (asset) => {
+    if (!acceptedAsset(asset)) throw new Error("Asset MIME/signature is unsupported or unsafe to serve.");
+    const sha256 = hash2(asset.content);
+    const content = assetBytes.get(sha256) ?? Buffer.from(asset.content);
+    assetBytes.set(sha256, content);
+    return { ...asset, content };
+  };
+  const assetRequests = createMigrationAssetCache();
   for (const record of checkpoint.records) for (const asset of record.assets ?? []) {
-    if (!assetRequests.has(asset.url)) assetRequests.set(asset.url, Promise.resolve({ url: asset.url, contentType: asset.contentType, content: Buffer.from(asset.base64, "base64") }));
+    if (!assetRequests.has(asset.url)) {
+      const content = assetBytes.get(asset.sha256) ?? Buffer.from(asset.base64, "base64");
+      assetBytes.set(asset.sha256, content);
+      assetRequests.remember(asset.url, { url: asset.url, contentType: asset.contentType, content });
+    }
   }
   let totalBytes = checkpoint.records.reduce((sum, record) => sum + (record.evidence?.byteLength ?? 0), 0) + [...assetPayloads.values()].reduce((sum, asset) => sum + Buffer.byteLength(asset.base64, "base64"), 0);
   let budgetExhausted = false;
@@ -14412,10 +14492,10 @@ ${difference}`;
         try {
           let assetRequest = assetRequests.get(url);
           if (!suppliedAssets.has(url) && !assetRequest && assetFetcher) {
-            assetRequest = assetFetcher(new URL(url), options.signal);
-            assetRequests.set(url, assetRequest);
+            assetRequest = assetRequests.load(url, () => assetFetcher(new URL(url), options.signal).then(retainAsset));
           }
-          const asset = suppliedAssets.get(url) ?? (assetRequest ? await assetRequest : null);
+          const supplied = suppliedAssets.get(url);
+          const asset = supplied ? retainAsset(supplied) : assetRequest ? await assetRequest : null;
           if (!asset) throw new Error("Capture host did not supply asset bytes or a binary fetch boundary.");
           const extension = acceptedAsset(asset);
           if (!extension) throw new Error("Asset MIME/signature is unsupported or unsafe to serve.");
@@ -14432,7 +14512,7 @@ ${difference}`;
           const receipt = { url, path, sha256, contentType: asset.contentType, base64: existing?.base64 ?? Buffer.from(asset.content).toString("base64") };
           record.assets.push(receipt);
           if (!existing) assetPayloads.set(sha256, receipt);
-          if (!assetRequests.has(url)) assetRequests.set(url, Promise.resolve(asset));
+          if (!assetRequests.has(url)) assetRequests.remember(url, asset);
           page.body = rewriteMarkdownDestinations(page.body, (destination) => canonical(destination, final) === url ? `/${path}${new URL(destination, final).hash}` : destination);
         } catch (error) {
           assetRequests.delete(url);
@@ -14548,7 +14628,13 @@ ${difference}`;
   const validation = validateUrlMigration(checkpoint, Boolean(options.signal?.aborted));
   validation.issues.push(...navigationIssues);
   if (navigationIssues.length && validation.status === "passed") validation.status = "partial";
-  const assets = new Map(checkpoint.records.flatMap((record) => (record.assets ?? []).map((asset) => [asset.path, { path: asset.path, content: Buffer.from(asset.base64, "base64") }])));
+  const assets = /* @__PURE__ */ new Map();
+  for (const record of checkpoint.records) for (const asset of record.assets ?? []) {
+    if (!assets.has(asset.path)) {
+      const content = assetBytes.get(asset.sha256) ?? Buffer.from(asset.base64, "base64");
+      assets.set(asset.path, { path: asset.path, content: Buffer.from(content.buffer, content.byteOffset, content.byteLength) });
+    }
+  }
   const bundle = {
     sourceUrl,
     sourceKind: "url",