@agentproto/catalog-sync 0.5.7 → 0.5.9

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { mkdirSync, writeFileSync, existsSync, readFileSync } from 'fs';
+import { existsSync, readFileSync, mkdirSync, writeFileSync } from 'fs';
 import { dirname, join, resolve } from 'path';
 import { fileURLToPath } from 'url';
 import { z } from 'zod';
@@ -14,22 +14,111 @@ import { ElevenLabsVoicesSnapshotSchema, mapElevenLabsVoices, MinimaxVoicesSnaps
 function defineGenerator(g) {
   return g;
 }
+
+// src/changelog.ts
+var RECORD_KEY_RE = /^ {2}"([^"]+)":\s*\{/gm;
+function extractRecordIds(source) {
+  const ids = /* @__PURE__ */ new Set();
+  for (const m of source.matchAll(RECORD_KEY_RE)) {
+    if (m[1] !== void 0) ids.add(m[1]);
+  }
+  return ids;
+}
+function diffModelIds(before, after) {
+  const beforeIds = before !== void 0 ? extractRecordIds(before) : /* @__PURE__ */ new Set();
+  const afterIds = extractRecordIds(after);
+  const added = [...afterIds].filter((id) => !beforeIds.has(id)).sort();
+  const removed = [...beforeIds].filter((id) => !afterIds.has(id)).sort();
+  return { added, removed };
+}
+function renderChangelogSection(date, entries) {
+  const nonEmpty = entries.filter((e) => e.added.length > 0 || e.removed.length > 0);
+  if (nonEmpty.length === 0) return "";
+  const lines = [`## ${date}`, ""];
+  for (const e of nonEmpty) {
+    lines.push(`### ${e.generator}`);
+    if (e.added.length > 0) lines.push(`- Added: ${e.added.join(", ")}`);
+    if (e.removed.length > 0) lines.push(`- Removed: ${e.removed.join(", ")}`);
+    lines.push("");
+  }
+  return lines.join("\n").replace(/\n+$/, "\n");
+}
+var CATALOG_CHANGELOG_HEADER = `# Catalog Changelog
+
+Human-readable log of model additions/removals detected by
+\`@agentproto/catalog-sync\` runs \u2014 NOT \`CHANGELOG.md\` next to it (that one is
+changesets-driven package-version history). A dated section is appended
+automatically here whenever a sync run adds or removes model ids; sections
+are appended at the END of the file (newest last). Pricing-only drift (no id
+added/removed) is not logged here \u2014 see the generated files' own git history
+for that.
+`;
+function appendChangelog(existing, date, entries) {
+  const section = renderChangelogSection(date, entries);
+  if (section === "") return void 0;
+  const base = existing !== void 0 && existing.length > 0 ? existing.replace(/\n+$/, "\n") : CATALOG_CHANGELOG_HEADER;
+  return `${base}
+${section}`;
+}
 var __dirname$1 = dirname(fileURLToPath(import.meta.url));
 function catalogSyncDir() {
   return resolve(__dirname$1, "..");
 }
+function ledgerAbsPath(ledgerId) {
+  return join(catalogSyncDir(), "ledger", `${ledgerId}.json`);
+}
+function ledgerRelPath(ledgerId) {
+  return `packages/catalog-sync/ledger/${ledgerId}.json`;
+}
+function readLedger(ledgerId) {
+  const path = ledgerAbsPath(ledgerId);
+  if (!existsSync(path)) return {};
+  try {
+    const parsed = JSON.parse(readFileSync(path, "utf8"));
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {};
+    return parsed;
+  } catch {
+    return {};
+  }
+}
+function serializeLedger(ledger) {
+  const sorted = {};
+  for (const id of Object.keys(ledger).sort()) sorted[id] = ledger[id];
+  return `${JSON.stringify(sorted, null, 2)}
+`;
+}
+function computeAddedAtLedger(currentIds, previous, createdAt, today) {
+  const next = { ...previous };
+  for (const id of currentIds) {
+    if (next[id] !== void 0) continue;
+    next[id] = createdAt[id] ?? today;
+  }
+  return next;
+}
+function todayIso() {
+  return (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
+}
+function isoDateFromUnixSeconds(seconds) {
+  return new Date(seconds * 1e3).toISOString().slice(0, 10);
+}
+
+// src/runner.ts
+var __dirname2 = dirname(fileURLToPath(import.meta.url));
+function catalogSyncDir2() {
+  return resolve(__dirname2, "..");
+}
 function snapshotPath(id) {
-  return join(catalogSyncDir(), "snapshots", `${id}.json`);
+  return join(catalogSyncDir2(), "snapshots", `${id}.json`);
 }
 function repoRoot() {
-  let dir = catalogSyncDir();
+  let dir = catalogSyncDir2();
   for (let i = 0; i < 12; i++) {
     if (existsSync(join(dir, "pnpm-workspace.yaml"))) return dir;
     const parent = dirname(dir);
     if (parent === dir) break;
     dir = parent;
   }
-  return dirname(dirname(catalogSyncDir()));
+  return dirname(dirname(catalogSyncDir2()));
 }
 function readIfExists(absPath) {
   if (!existsSync(absPath)) return void 0;
@@ -51,8 +140,13 @@ function resolveHeaders(headers) {
   }
   return { headers: out, missing };
 }
+var CATALOG_CHANGELOG_PATH = "packages/model-catalog/CATALOG-CHANGELOG.md";
+function isChangelogEligible(relPath) {
+  return relPath.endsWith(".generated.ts");
+}
 async function runGenerators(gens, opts) {
   const files = {};
+  const pathToGenerator = {};
   for (const gen of gens) {
     const ctx = {
       refresh: opts.refresh,
@@ -99,10 +193,12 @@ async function runGenerators(gens, opts) {
     const out = await gen.generate(ctx);
     for (const [path, content] of Object.entries(out)) {
       files[path] = content;
+      pathToGenerator[path] = gen.name;
     }
   }
   const root = repoRoot();
   const changed = [];
+  const changelogEntries = [];
   for (const [relPath, content] of Object.entries(files)) {
     const abs = join(root, relPath);
     const existing = readIfExists(abs);
@@ -113,6 +209,23 @@ async function runGenerators(gens, opts) {
         writeFileSync(abs, content, "utf8");
       }
     }
+    if (isChangelogEligible(relPath)) {
+      const { added, removed } = diffModelIds(existing, content);
+      if (added.length > 0 || removed.length > 0) {
+        changelogEntries.push({ generator: pathToGenerator[relPath] ?? relPath, added, removed });
+      }
+    }
+  }
+  const changelogAbs = join(root, CATALOG_CHANGELOG_PATH);
+  const changelogExisting = readIfExists(changelogAbs);
+  const newChangelog = appendChangelog(changelogExisting, todayIso(), changelogEntries);
+  if (newChangelog !== void 0) {
+    files[CATALOG_CHANGELOG_PATH] = newChangelog;
+    changed.push(CATALOG_CHANGELOG_PATH);
+    if (opts.write) {
+      mkdirSync(dirname(changelogAbs), { recursive: true });
+      writeFileSync(changelogAbs, newChangelog, "utf8");
+    }
   }
   return { files, changed };
 }
@@ -126,7 +239,8 @@ var PricingSchema = z.object({
 }).passthrough();
 var ModelSchema = z.object({
   id: z.string(),
-  pricing: PricingSchema.optional()
+  pricing: PricingSchema.optional(),
+  created: z.number().optional()
 }).passthrough();
 var SnapshotSchema = z.object({
   data: z.array(ModelSchema)
@@ -159,6 +273,9 @@ function serializeEntry(e) {
   if (e.cacheWriteMultiplier !== void 0) {
     fields.push(`cacheWriteMultiplier: ${fmt(e.cacheWriteMultiplier)}`);
   }
+  if (e.addedAt !== void 0) {
+    fields.push(`addedAt: ${JSON.stringify(e.addedAt)}`);
+  }
   fields.push(`vendor: ${JSON.stringify(e.vendor)}`);
   fields.push(`provider: ${JSON.stringify(e.provider)}`);
   return `{
@@ -175,6 +292,9 @@ function serializeFile(entries) {
     "// Pricing carries provider USD (inputPer1M / outputPer1M) plus cache",
     "// multipliers (cacheReadMultiplier / cacheWriteMultiplier) derived from the",
     "// source's input_cache_read / input_cache_write per-token fields when present.",
+    "// addedAt is the ISO date this id was first seen by a sync run \u2014 backfilled",
+    "// from the source's own `created` timestamp, then NEVER mutated; see",
+    "// packages/catalog-sync/src/added-at.ts and the package README.",
     "",
     'import type { LLMPricing } from "./catalog.js"',
     "",
@@ -194,11 +314,13 @@ function serializeFile(entries) {
   lines.push("");
   return lines.join("\n");
 }
+var LEDGER_ID = "llm-openrouter";
 async function generate(ctx) {
   const src = sources[0];
   if (!src) throw new Error("llm:openrouter: no source configured");
   const parsed = SnapshotSchema.parse(await ctx.fetchSource(src));
   const entries = {};
+  const createdAt = {};
   for (const model of parsed.data) {
     const pricing = model.pricing;
     if (!pricing) continue;
@@ -221,8 +343,19 @@ async function generate(ctx) {
       entry.cacheWriteMultiplier = round6(cacheWritePer1M / inputPer1M);
     }
     entries[model.id] = entry;
+    if (model.created !== void 0) createdAt[model.id] = isoDateFromUnixSeconds(model.created);
   }
-  return { [OUTPUT_PATH]: serializeFile(entries) };
+  const ledger = computeAddedAtLedger(
+    Object.keys(entries),
+    readLedger(LEDGER_ID),
+    createdAt,
+    todayIso()
+  );
+  for (const [id, entry] of Object.entries(entries)) entry.addedAt = ledger[id];
+  return {
+    [OUTPUT_PATH]: serializeFile(entries),
+    [ledgerRelPath(LEDGER_ID)]: serializeLedger(ledger)
+  };
 }
 var sources = [{ id: "llm-openrouter", url: OPENROUTER_MODELS_URL }];
 var llmOpenRouterGenerator = defineGenerator({
@@ -238,7 +371,8 @@ var ModelSchema2 = z.object({
   input_price: z.number().optional(),
   output_price: z.number().optional(),
   cached_price: z.number().optional(),
-  supports_caching: z.boolean().optional()
+  supports_caching: z.boolean().optional(),
+  created: z.number().optional()
 }).passthrough();
 var SnapshotSchema2 = z.object({
   data: z.array(ModelSchema2)
@@ -267,6 +401,9 @@ function serializeEntry2(e) {
   if (e.cacheReadMultiplier !== void 0) {
     fields.push(`cacheReadMultiplier: ${fmt2(e.cacheReadMultiplier)}`);
   }
+  if (e.addedAt !== void 0) {
+    fields.push(`addedAt: ${JSON.stringify(e.addedAt)}`);
+  }
   fields.push(`vendor: ${JSON.stringify(e.vendor)}`);
   fields.push(`provider: ${JSON.stringify(e.provider)}`);
   return `{
@@ -284,6 +421,9 @@ function serializeFile2(entries) {
     "// multiplier (cacheReadMultiplier) derived from the source's cached_price",
     "// per-token field when `supports_caching` is true. Requesty has no",
     "// cache-write price, so cacheWriteMultiplier is never emitted.",
+    "// addedAt is the ISO date this id was first seen by a sync run \u2014 backfilled",
+    "// from the source's own `created` timestamp, then NEVER mutated; see",
+    "// packages/catalog-sync/src/added-at.ts and the package README.",
     "",
     'import type { LLMPricing } from "./catalog.js"',
     "",
@@ -303,11 +443,13 @@ function serializeFile2(entries) {
   lines.push("");
   return lines.join("\n");
 }
+var LEDGER_ID2 = "llm-requesty";
 async function generate2(ctx) {
   const src = sources2[0];
   if (!src) throw new Error("llm:requesty: no source configured");
   const parsed = SnapshotSchema2.parse(await ctx.fetchSource(src));
   const entries = {};
+  const createdAt = {};
   for (const model of parsed.data) {
     const inputPer1M = per1m2(model.input_price);
     const outputPer1M = per1m2(model.output_price);
@@ -326,8 +468,19 @@ async function generate2(ctx) {
       }
     }
     entries[model.id] = entry;
+    if (model.created !== void 0) createdAt[model.id] = isoDateFromUnixSeconds(model.created);
   }
-  return { [OUTPUT_PATH2]: serializeFile2(entries) };
+  const ledger = computeAddedAtLedger(
+    Object.keys(entries),
+    readLedger(LEDGER_ID2),
+    createdAt,
+    todayIso()
+  );
+  for (const [id, entry] of Object.entries(entries)) entry.addedAt = ledger[id];
+  return {
+    [OUTPUT_PATH2]: serializeFile2(entries),
+    [ledgerRelPath(LEDGER_ID2)]: serializeLedger(ledger)
+  };
 }
 var sources2 = [
   {
@@ -844,22 +997,22 @@ var imageReplicate = defineGenerator({
     return { [outputPath]: lines.join("\n") };
   }
 });
-var __dirname2 = dirname(fileURLToPath(import.meta.url));
-function catalogSyncDir2() {
-  return resolve(__dirname2, "..");
+var __dirname3 = dirname(fileURLToPath(import.meta.url));
+function catalogSyncDir3() {
+  return resolve(__dirname3, "..");
 }
 function snapshotPath2(id) {
-  return join(catalogSyncDir2(), "snapshots", `${id}.json`);
+  return join(catalogSyncDir3(), "snapshots", `${id}.json`);
 }
 function repoRoot2() {
-  let dir = catalogSyncDir2();
+  let dir = catalogSyncDir3();
   for (let i = 0; i < 12; i++) {
     if (existsSync(join(dir, "pnpm-workspace.yaml"))) return dir;
     const parent = dirname(dir);
     if (parent === dir) break;
     dir = parent;
   }
-  return dirname(dirname(catalogSyncDir2()));
+  return dirname(dirname(catalogSyncDir3()));
 }
 function readIfExists2(absPath) {
   if (!existsSync(absPath)) return void 0;
@@ -1003,6 +1156,6 @@ var OPENAI_LLM_SOURCE = {
 };
 var OPENAI_SOURCES = [OPENAI_LLM_SOURCE];
 
-export { OPENAI_LLM_SOURCE, OPENAI_SOURCES, defineGenerator, imageReplicate, llmHuggingfaceGenerator, llmOpenRouterGenerator, llmRequestyGenerator, refreshSources, reviewedRefresh, runGenerators, voiceElevenlabs, voiceMinimax };
+export { CATALOG_CHANGELOG_HEADER, CATALOG_CHANGELOG_PATH, OPENAI_LLM_SOURCE, OPENAI_SOURCES, appendChangelog, computeAddedAtLedger, defineGenerator, diffModelIds, extractRecordIds, imageReplicate, isoDateFromUnixSeconds, ledgerAbsPath, ledgerRelPath, llmHuggingfaceGenerator, llmOpenRouterGenerator, llmRequestyGenerator, readLedger, refreshSources, renderChangelogSection, reviewedRefresh, runGenerators, serializeLedger, todayIso, voiceElevenlabs, voiceMinimax };
 //# sourceMappingURL=index.mjs.map
 //# sourceMappingURL=index.mjs.map