@x12i/ai-tools 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +114 -12
- package/dist/{AiModelsCatalogClient-CSVlKql5.d.cts → AiModelsCatalogClient-CNeqFiFs.d.cts} +2 -1
- package/dist/{AiModelsCatalogClient-B-dNLXX0.d.ts → AiModelsCatalogClient-nwFoEaqL.d.ts} +2 -1
- package/dist/aliases/index.d.cts +4 -3
- package/dist/aliases/index.d.ts +4 -3
- package/dist/catalog/index.cjs +30 -0
- package/dist/catalog/index.cjs.map +1 -0
- package/dist/catalog/index.d.cts +100 -0
- package/dist/catalog/index.d.ts +100 -0
- package/dist/catalog/index.js +30 -0
- package/dist/catalog/index.js.map +1 -0
- package/dist/catalox/index.cjs +2 -2
- package/dist/catalox/index.d.cts +7 -19
- package/dist/catalox/index.d.ts +7 -19
- package/dist/catalox/index.js +1 -1
- package/dist/chunk-C3H7RTFR.cjs +1 -0
- package/dist/chunk-C3H7RTFR.cjs.map +1 -0
- package/dist/{chunk-ONA73BU6.cjs → chunk-DKHGWHXP.cjs} +21 -12
- package/dist/chunk-DKHGWHXP.cjs.map +1 -0
- package/dist/{chunk-HHNHWYTP.cjs → chunk-FGP3QXWL.cjs} +94 -36
- package/dist/chunk-FGP3QXWL.cjs.map +1 -0
- package/dist/chunk-HS74X2OJ.cjs +172 -0
- package/dist/chunk-HS74X2OJ.cjs.map +1 -0
- package/dist/chunk-HYGXZY25.js +163 -0
- package/dist/chunk-HYGXZY25.js.map +1 -0
- package/dist/chunk-M5TMA73F.js +1 -0
- package/dist/chunk-M5TMA73F.js.map +1 -0
- package/dist/chunk-MX3AMQFC.js +172 -0
- package/dist/chunk-MX3AMQFC.js.map +1 -0
- package/dist/{chunk-MLRHYOCD.js → chunk-VRFVF5RH.js} +21 -12
- package/dist/chunk-VRFVF5RH.js.map +1 -0
- package/dist/cli/index.cjs +133 -30
- package/dist/cli/index.cjs.map +1 -1
- package/dist/cli/index.js +134 -31
- package/dist/cli/index.js.map +1 -1
- package/dist/cost/index.d.cts +4 -3
- package/dist/cost/index.d.ts +4 -3
- package/dist/index.cjs +17 -6
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +10 -16
- package/dist/index.d.ts +10 -16
- package/dist/index.js +22 -11
- package/dist/{modelNameResolver-Bn8QnkSj.d.ts → modelNameResolver-D9V_GfUK.d.cts} +3 -27
- package/dist/{modelNameResolver-bZD-eBSJ.d.cts → modelNameResolver-DqFt7g6W.d.ts} +3 -27
- package/dist/models/index.d.cts +3 -2
- package/dist/models/index.d.ts +3 -2
- package/dist/sync/index.cjs +3 -3
- package/dist/sync/index.d.cts +6 -3
- package/dist/sync/index.d.ts +6 -3
- package/dist/sync/index.js +2 -2
- package/dist/syncAiModelsCatalog-CnXRLm2c.d.cts +32 -0
- package/dist/syncAiModelsCatalog-DpkN_w7S.d.ts +32 -0
- package/dist/types-BYXnCvKx.d.cts +137 -0
- package/dist/types-BYXnCvKx.d.ts +137 -0
- package/dist/types-CX6QFNNy.d.cts +144 -0
- package/dist/types-CuiPDcVs.d.ts +144 -0
- package/dist/upsertAiModelRecord-C831wOIF.d.ts +35 -0
- package/dist/upsertAiModelRecord-CjY-sny0.d.cts +35 -0
- package/package.json +8 -1
- package/dist/chunk-HHNHWYTP.cjs.map +0 -1
- package/dist/chunk-ML2FRR4L.js +0 -105
- package/dist/chunk-ML2FRR4L.js.map +0 -1
- package/dist/chunk-MLRHYOCD.js.map +0 -1
- package/dist/chunk-ONA73BU6.cjs.map +0 -1
- package/dist/types-DdGB3YaA.d.cts +0 -278
- package/dist/types-DdGB3YaA.d.ts +0 -278
|
@@ -38,61 +38,119 @@ function buildIndexed(record) {
|
|
|
38
38
|
syncedAt: record.syncedAt
|
|
39
39
|
};
|
|
40
40
|
}
|
|
41
|
-
|
|
42
|
-
const {
|
|
43
|
-
|
|
44
|
-
const layout = await _firebase.resolveNativeItemsLayout.call(void 0, firestore, catalogId);
|
|
45
|
-
const col = layout === "legacy" ? _firebase.legacyNativeItemsCollectionRef.call(void 0, firestore, catalogId) : _firebase.flatNativeItemsCollectionRef.call(void 0, firestore, catalogId);
|
|
46
|
-
const storageDocId = encodeModelFirestoreDocId(record.modelId);
|
|
47
|
-
const existing = await col.doc(storageDocId).get();
|
|
48
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
49
|
-
const data = sanitizeForFirestore(record);
|
|
50
|
-
const payload = sanitizeForFirestore({
|
|
41
|
+
function buildCatalogItemUpsertPayload(record, existing, options) {
|
|
42
|
+
const { catalogId, appId, now } = options;
|
|
43
|
+
return sanitizeForFirestore({
|
|
51
44
|
itemId: record.modelId,
|
|
52
45
|
catalogId,
|
|
53
46
|
appScopedOwnerId: appId,
|
|
54
47
|
indexed: buildIndexed(record),
|
|
55
|
-
data,
|
|
48
|
+
data: sanitizeForFirestore(record),
|
|
56
49
|
metadata: {
|
|
57
|
-
createdAt: String(_nullishCoalesce(_optionalChain([existing, '
|
|
50
|
+
createdAt: String(_nullishCoalesce(_optionalChain([existing, 'optionalAccess', _ => _.metadata, 'optionalAccess', _2 => _2.createdAt]), () => ( now))),
|
|
58
51
|
lastUpdate: now,
|
|
59
52
|
domainIds: [],
|
|
60
53
|
agentIds: []
|
|
61
54
|
},
|
|
62
|
-
version: (_nullishCoalesce(_optionalChain([existing, '
|
|
55
|
+
version: (_nullishCoalesce(_optionalChain([existing, 'optionalAccess', _3 => _3.version]), () => ( 0))) + 1
|
|
63
56
|
});
|
|
57
|
+
}
|
|
58
|
+
var EXISTING_READ_BATCH = 50;
|
|
59
|
+
async function loadExistingByStorageId(col, records) {
|
|
60
|
+
const map = /* @__PURE__ */ new Map();
|
|
61
|
+
for (let i = 0; i < records.length; i += EXISTING_READ_BATCH) {
|
|
62
|
+
const slice = records.slice(i, i + EXISTING_READ_BATCH);
|
|
63
|
+
const snaps = await Promise.all(
|
|
64
|
+
slice.map((r) => col.doc(encodeModelFirestoreDocId(r.modelId)).get())
|
|
65
|
+
);
|
|
66
|
+
for (let j = 0; j < slice.length; j++) {
|
|
67
|
+
const storageDocId = encodeModelFirestoreDocId(slice[j].modelId);
|
|
68
|
+
const data = snaps[j].data();
|
|
69
|
+
if (data) map.set(storageDocId, data);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return map;
|
|
73
|
+
}
|
|
74
|
+
async function upsertAiModelRecord(options) {
|
|
75
|
+
const { record, catalogId, context, firestore } = options;
|
|
76
|
+
const appId = _nullishCoalesce(context.appId, () => ( "ai-tools"));
|
|
77
|
+
const layout = await _firebase.resolveNativeItemsLayout.call(void 0, firestore, catalogId);
|
|
78
|
+
const col = layout === "legacy" ? _firebase.legacyNativeItemsCollectionRef.call(void 0, firestore, catalogId) : _firebase.flatNativeItemsCollectionRef.call(void 0, firestore, catalogId);
|
|
79
|
+
const storageDocId = encodeModelFirestoreDocId(record.modelId);
|
|
80
|
+
const existing = (await col.doc(storageDocId).get()).data();
|
|
81
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
82
|
+
const payload = buildCatalogItemUpsertPayload(record, existing, { catalogId, appId, now });
|
|
64
83
|
await col.doc(storageDocId).set(payload, { merge: true });
|
|
65
84
|
}
|
|
66
85
|
var BATCH_SIZE = 400;
|
|
67
|
-
|
|
86
|
+
var DEFAULT_CHUNK_RETRIES = 3;
|
|
87
|
+
var RETRY_BASE_MS = 500;
|
|
88
|
+
function sleep(ms) {
|
|
89
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
90
|
+
}
|
|
91
|
+
async function commitChunkWithRetry(firestore, col, chunk, existingByStorageId, options) {
|
|
92
|
+
const { catalogId, appId, now, maxChunkRetries } = options;
|
|
93
|
+
for (let attempt = 0; attempt < maxChunkRetries; attempt++) {
|
|
94
|
+
try {
|
|
95
|
+
const batch = firestore.batch();
|
|
96
|
+
for (const record of chunk) {
|
|
97
|
+
const storageDocId = encodeModelFirestoreDocId(record.modelId);
|
|
98
|
+
const existing = existingByStorageId.get(storageDocId);
|
|
99
|
+
const payload = buildCatalogItemUpsertPayload(record, existing, { catalogId, appId, now });
|
|
100
|
+
batch.set(col.doc(storageDocId), payload, { merge: true });
|
|
101
|
+
}
|
|
102
|
+
await batch.commit();
|
|
103
|
+
return { upserted: chunk.length, failed: [] };
|
|
104
|
+
} catch (e) {
|
|
105
|
+
if (attempt < maxChunkRetries - 1) {
|
|
106
|
+
await sleep(RETRY_BASE_MS * 2 ** attempt);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
const failed = [];
|
|
111
|
+
let upserted = 0;
|
|
112
|
+
for (const record of chunk) {
|
|
113
|
+
try {
|
|
114
|
+
const storageDocId = encodeModelFirestoreDocId(record.modelId);
|
|
115
|
+
const existing = existingByStorageId.get(storageDocId);
|
|
116
|
+
const payload = buildCatalogItemUpsertPayload(record, existing, { catalogId, appId, now });
|
|
117
|
+
await col.doc(storageDocId).set(payload, { merge: true });
|
|
118
|
+
upserted++;
|
|
119
|
+
} catch (error) {
|
|
120
|
+
failed.push({
|
|
121
|
+
modelId: record.modelId,
|
|
122
|
+
error: error instanceof Error ? error.message : String(error)
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return { upserted, failed };
|
|
127
|
+
}
|
|
128
|
+
async function batchUpsertAiModelRecords(firestore, catalogId, context, records, batchOptions = {}) {
|
|
68
129
|
const layout = await _firebase.resolveNativeItemsLayout.call(void 0, firestore, catalogId);
|
|
69
130
|
const col = layout === "legacy" ? _firebase.legacyNativeItemsCollectionRef.call(void 0, firestore, catalogId) : _firebase.flatNativeItemsCollectionRef.call(void 0, firestore, catalogId);
|
|
70
131
|
const appId = _nullishCoalesce(context.appId, () => ( "ai-tools"));
|
|
71
132
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
133
|
+
const maxChunkRetries = _nullishCoalesce(batchOptions.maxChunkRetries, () => ( DEFAULT_CHUNK_RETRIES));
|
|
134
|
+
const total = records.length;
|
|
135
|
+
_optionalChain([batchOptions, 'access', _4 => _4.onProgress, 'optionalCall', _5 => _5({ completed: 0, total, phase: "loading-existing" })]);
|
|
136
|
+
const existingByStorageId = await loadExistingByStorageId(col, records);
|
|
137
|
+
let completed = 0;
|
|
138
|
+
let upserted = 0;
|
|
139
|
+
const failed = [];
|
|
72
140
|
for (let i = 0; i < records.length; i += BATCH_SIZE) {
|
|
73
141
|
const chunk = records.slice(i, i + BATCH_SIZE);
|
|
74
|
-
const
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
metadata: {
|
|
85
|
-
createdAt: now,
|
|
86
|
-
lastUpdate: now,
|
|
87
|
-
domainIds: [],
|
|
88
|
-
agentIds: []
|
|
89
|
-
},
|
|
90
|
-
version: 1
|
|
91
|
-
});
|
|
92
|
-
batch.set(col.doc(storageDocId), payload, { merge: true });
|
|
93
|
-
}
|
|
94
|
-
await batch.commit();
|
|
142
|
+
const chunkResult = await commitChunkWithRetry(firestore, col, chunk, existingByStorageId, {
|
|
143
|
+
catalogId,
|
|
144
|
+
appId,
|
|
145
|
+
now,
|
|
146
|
+
maxChunkRetries
|
|
147
|
+
});
|
|
148
|
+
upserted += chunkResult.upserted;
|
|
149
|
+
failed.push(...chunkResult.failed);
|
|
150
|
+
completed += chunk.length;
|
|
151
|
+
_optionalChain([batchOptions, 'access', _6 => _6.onProgress, 'optionalCall', _7 => _7({ completed, total, phase: "writing" })]);
|
|
95
152
|
}
|
|
153
|
+
return { upserted, failed };
|
|
96
154
|
}
|
|
97
155
|
|
|
98
156
|
|
|
@@ -102,4 +160,4 @@ async function batchUpsertAiModelRecords(firestore, catalogId, context, records)
|
|
|
102
160
|
|
|
103
161
|
|
|
104
162
|
exports.encodeModelFirestoreDocId = encodeModelFirestoreDocId; exports.decodeModelFirestoreDocId = decodeModelFirestoreDocId; exports.sanitizeForFirestore = sanitizeForFirestore; exports.upsertAiModelRecord = upsertAiModelRecord; exports.batchUpsertAiModelRecords = batchUpsertAiModelRecords;
|
|
105
|
-
//# sourceMappingURL=chunk-
|
|
163
|
+
//# sourceMappingURL=chunk-FGP3QXWL.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/ami/Documents/prometheus/x12i/ai-tools/dist/chunk-FGP3QXWL.cjs","../src/catalox/modelDocId.ts","../src/catalox/upsertAiModelRecord.ts"],"names":[],"mappings":"AAAA;ACIA,IAAM,GAAA,EAAK,KAAA;AACX,IAAM,aAAA,EAAe,IAAA;AAEd,SAAS,yBAAA,CAA0B,OAAA,EAAyB;AACjE,EAAA,GAAA,CAAI,CAAC,OAAA,CAAQ,QAAA,CAAS,GAAG,CAAA,EAAG,OAAO,OAAA;AACnC,EAAA,OAAO,CAAA,EAAA;AACT;AAEgB;AACR,EAAA;AACF,EAAA;AACF,IAAA;AACF,EAAA;AACO,EAAA;AACT;ADJU;AACA;AEbV;AACE;AACA;AACA;AACK;AAKS;AACP,EAAA;AACT;AAES;AACA,EAAA;AACL,IAAA;AACA,IAAA;AACM,IAAA;AACN,IAAA;AACA,IAAA;AACA,IAAA;AACA,IAAA;AACA,IAAA;AACA,IAAA;AACA,IAAA;AACA,IAAA;AACA,IAAA;AACF,EAAA;AACF;AAQgB;AAKN,EAAA;AACD,EAAA;AACL,IAAA;AACA,IAAA;AACA,IAAA;AACA,IAAA;AACM,IAAA;AACN,IAAA;AACE,MAAA;AACA,MAAA;AACA,MAAA;AACA,MAAA;AACF,IAAA;AACA,IAAA;AACD,EAAA;AACH;AAEM;AAEN;AAIQ,EAAA;AACN,EAAA;AACQ,IAAA;AACA,IAAA;AACJ,MAAA;AACF,IAAA;AACA,IAAA;AACE,MAAA;AACA,MAAA;AACI,MAAA;AACN,IAAA;AACF,EAAA;AACO,EAAA;AACT;AAYA;AACU,EAAA;AACF,EAAA;AAEA,EAAA;AACA,EAAA;AAKA,EAAA;AACA,EAAA;AACA,EAAA;AACA,EAAA;AAEA,EAAA;AACR;AAEM;AACA;AACA;AAkBG;AACA,EAAA;AACT;AAEA;AAYU,EAAA;AACR,EAAA;AACM,IAAA;AACF,MAAA;AACA,MAAA;AACE,QAAA;AACA,QAAA;AACA,QAAA;AACA,QAAA;AACF,MAAA;AACA,MAAA;AACA,MAAA;AACF,IAAA;AACM,MAAA;AACF,QAAA;AACF,MAAA;AACF,IAAA;AACF,EAAA;AAEM,EAAA;AACF,EAAA;AAEJ,EAAA;AACM,IAAA;AACF,MAAA;AACA,MAAA;AACA,MAAA;AACA,MAAA;AACA,MAAA;AACF,IAAA;AACE,MAAA;AACE,QAAA;AACA,QAAA;AACD,MAAA;AACH,IAAA;AACF,EAAA;AAEO,EAAA;AACT;AAEA;AAOQ,EAAA;AACA,EAAA;AAKA,EAAA;AACA,EAAA;AACA,EAAA;AACA,EAAA;AAEN,kBAAA;AACM,EAAA;AAEF,EAAA;AACA,EAAA;AACE,EAAA;AAEN,EAAA;AACQ,IAAA;AACA,IAAA;AACJ,MAAA;AACA,MAAA;AACA,MAAA;AACA,MAAA;AACD,IAAA;AACD,IAAA;AACA,IAAA;AACA,IAAA;AACA,oBAAA;AACF,EAAA;AAEO,EAAA;AACT;AFvEU;AACA;AACA;AACA;AACA;AACA;AACA;AACA","file":"/Users/ami/Documents/prometheus/x12i/ai-tools/dist/chunk-FGP3QXWL.cjs","sourcesContent":[null,"/**\n * Firestore document ids cannot contain `/`. OpenRouter model ids use `provider/model`.\n * We encode global item doc ids as `cx~m~<base64url(modelId)>` while keeping logical `modelId` in data.\n */\nconst CX = \"cx~\";\nconst MODEL_MARKER = \"m~\";\n\nexport function encodeModelFirestoreDocId(modelId: string): string {\n if (!modelId.includes(\"/\")) return modelId;\n return `${CX}${MODEL_MARKER}${Buffer.from(modelId, \"utf8\").toString(\"base64url\")}`;\n}\n\nexport function decodeModelFirestoreDocId(docId: string): string {\n const prefix = `${CX}${MODEL_MARKER}`;\n if (docId.startsWith(prefix)) {\n return Buffer.from(docId.slice(prefix.length), \"base64url\").toString(\"utf8\");\n }\n return docId;\n}\n\nexport function modelIdNeedsEncodedDocId(modelId: string): boolean {\n return modelId.includes(\"/\");\n}\n","import type { Firestore } from \"firebase-admin/firestore\";\nimport type { CataloxContext } from \"@x12i/catalox\";\nimport {\n flatNativeItemsCollectionRef,\n legacyNativeItemsCollectionRef,\n resolveNativeItemsLayout,\n} from \"@x12i/catalox/firebase\";\nimport type { AiModelRecord } from \"../models/types.js\";\nimport { encodeModelFirestoreDocId } from \"./modelDocId.js\";\n\n/** Firestore rejects `undefined` — strip before write. */\nexport function sanitizeForFirestore<T>(value: T): T {\n return JSON.parse(JSON.stringify(value)) as T;\n}\n\nfunction buildIndexed(record: AiModelRecord): Record<string, string | number | boolean> {\n return {\n modelId: record.modelId,\n providerId: record.providerId,\n name: record.name,\n canonicalSlug: record.canonicalSlug,\n contextLength: record.contextLength,\n status: record.status,\n primaryOutputModality: record.primaryOutputModality,\n supportsTools: record.supportsTools,\n supportsReasoning: record.supportsReasoning,\n supportsStreaming: record.supportsStreaming,\n isModerated: record.isModerated,\n syncedAt: record.syncedAt,\n };\n}\n\ntype ExistingCatalogDoc = {\n metadata?: { createdAt?: string };\n version?: number;\n};\n\n/** @internal Exported for unit tests. */\nexport function buildCatalogItemUpsertPayload(\n record: AiModelRecord,\n existing: ExistingCatalogDoc | undefined,\n options: { catalogId: string; appId: string; now: string },\n) {\n const { catalogId, appId, now } = options;\n return sanitizeForFirestore({\n itemId: record.modelId,\n catalogId,\n appScopedOwnerId: appId,\n indexed: buildIndexed(record),\n data: sanitizeForFirestore(record),\n metadata: {\n createdAt: String(existing?.metadata?.createdAt ?? now),\n lastUpdate: now,\n domainIds: [],\n agentIds: [],\n },\n version: (existing?.version ?? 0) + 1,\n });\n}\n\nconst EXISTING_READ_BATCH = 50;\n\nasync function loadExistingByStorageId(\n col: ReturnType<typeof flatNativeItemsCollectionRef>,\n records: AiModelRecord[],\n): Promise<Map<string, ExistingCatalogDoc>> {\n const map = new Map<string, ExistingCatalogDoc>();\n for (let i = 0; i < records.length; i += EXISTING_READ_BATCH) {\n const slice = records.slice(i, i + EXISTING_READ_BATCH);\n const snaps = await Promise.all(\n slice.map((r) => col.doc(encodeModelFirestoreDocId(r.modelId)).get()),\n );\n for (let j = 0; j < slice.length; j++) {\n const storageDocId = encodeModelFirestoreDocId(slice[j]!.modelId);\n const data = snaps[j]!.data() as ExistingCatalogDoc | undefined;\n if (data) map.set(storageDocId, data);\n }\n }\n return map;\n}\n\nexport type UpsertAiModelRecordOptions = {\n firestore: Firestore;\n context: CataloxContext;\n catalogId: string;\n record: AiModelRecord;\n};\n\n/**\n * Upserts one ai-models row via Firestore (safe doc ids for `provider/model` ids).\n */\nexport async function upsertAiModelRecord(options: UpsertAiModelRecordOptions): Promise<void> {\n const { record, catalogId, context, firestore } = options;\n const appId = context.appId ?? \"ai-tools\";\n\n const layout = await resolveNativeItemsLayout(firestore, catalogId);\n const col =\n layout === \"legacy\"\n ? legacyNativeItemsCollectionRef(firestore, catalogId)\n : flatNativeItemsCollectionRef(firestore, catalogId);\n\n const storageDocId = encodeModelFirestoreDocId(record.modelId);\n const existing = (await col.doc(storageDocId).get()).data() as ExistingCatalogDoc | undefined;\n const now = new Date().toISOString();\n const payload = buildCatalogItemUpsertPayload(record, existing, { catalogId, appId, now });\n\n await col.doc(storageDocId).set(payload, { merge: true });\n}\n\nconst BATCH_SIZE = 400;\nconst DEFAULT_CHUNK_RETRIES = 3;\nconst RETRY_BASE_MS = 500;\n\nexport type BatchUpsertProgress = {\n completed: number;\n total: number;\n phase: \"loading-existing\" | \"writing\";\n};\n\nexport type BatchUpsertAiModelRecordsOptions = {\n onProgress?: (progress: BatchUpsertProgress) => void;\n maxChunkRetries?: number;\n};\n\nexport type BatchUpsertResult = {\n upserted: number;\n failed: Array<{ modelId: string; error: string }>;\n};\n\nfunction sleep(ms: number): Promise<void> {\n return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\nasync function commitChunkWithRetry(\n firestore: Firestore,\n col: ReturnType<typeof flatNativeItemsCollectionRef>,\n chunk: AiModelRecord[],\n existingByStorageId: Map<string, ExistingCatalogDoc>,\n options: {\n catalogId: string;\n appId: string;\n now: string;\n maxChunkRetries: number;\n },\n): Promise<{ upserted: number; failed: Array<{ modelId: string; error: string }> }> {\n const { catalogId, appId, now, maxChunkRetries } = options;\n for (let attempt = 0; attempt < maxChunkRetries; attempt++) {\n try {\n const batch = firestore.batch();\n for (const record of chunk) {\n const storageDocId = encodeModelFirestoreDocId(record.modelId);\n const existing = existingByStorageId.get(storageDocId);\n const payload = buildCatalogItemUpsertPayload(record, existing, { catalogId, appId, now });\n batch.set(col.doc(storageDocId), payload, { merge: true });\n }\n await batch.commit();\n return { upserted: chunk.length, failed: [] };\n } catch {\n if (attempt < maxChunkRetries - 1) {\n await sleep(RETRY_BASE_MS * 2 ** attempt);\n }\n }\n }\n\n const failed: Array<{ modelId: string; error: string }> = [];\n let upserted = 0;\n\n for (const record of chunk) {\n try {\n const storageDocId = encodeModelFirestoreDocId(record.modelId);\n const existing = existingByStorageId.get(storageDocId);\n const payload = buildCatalogItemUpsertPayload(record, existing, { catalogId, appId, now });\n await col.doc(storageDocId).set(payload, { merge: true });\n upserted++;\n } catch (error) {\n failed.push({\n modelId: record.modelId,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n\n return { upserted, failed };\n}\n\nexport async function batchUpsertAiModelRecords(\n firestore: Firestore,\n catalogId: string,\n context: CataloxContext,\n records: AiModelRecord[],\n batchOptions: BatchUpsertAiModelRecordsOptions = {},\n): Promise<BatchUpsertResult> {\n const layout = await resolveNativeItemsLayout(firestore, catalogId);\n const col =\n layout === \"legacy\"\n ? legacyNativeItemsCollectionRef(firestore, catalogId)\n : flatNativeItemsCollectionRef(firestore, catalogId);\n\n const appId = context.appId ?? \"ai-tools\";\n const now = new Date().toISOString();\n const maxChunkRetries = batchOptions.maxChunkRetries ?? DEFAULT_CHUNK_RETRIES;\n const total = records.length;\n\n batchOptions.onProgress?.({ completed: 0, total, phase: \"loading-existing\" });\n const existingByStorageId = await loadExistingByStorageId(col, records);\n\n let completed = 0;\n let upserted = 0;\n const failed: Array<{ modelId: string; error: string }> = [];\n\n for (let i = 0; i < records.length; i += BATCH_SIZE) {\n const chunk = records.slice(i, i + BATCH_SIZE);\n const chunkResult = await commitChunkWithRetry(firestore, col, chunk, existingByStorageId, {\n catalogId,\n appId,\n now,\n maxChunkRetries,\n });\n upserted += chunkResult.upserted;\n failed.push(...chunkResult.failed);\n completed += chunk.length;\n batchOptions.onProgress?.({ completed, total, phase: \"writing\" });\n }\n\n return { upserted, failed };\n}\n"]}
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _nullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return rhsFn(); } } async function _asyncNullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return await rhsFn(); } } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
var _chunkDKHGWHXPcjs = require('./chunk-DKHGWHXP.cjs');
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
var _chunkFGP3QXWLcjs = require('./chunk-FGP3QXWL.cjs');
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
var _chunkF2F4UEFDcjs = require('./chunk-F2F4UEFD.cjs');
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
var _chunkTF4L2NECcjs = require('./chunk-TF4L2NEC.cjs');
|
|
19
|
+
|
|
20
|
+
// src/catalog/verifyAiModelsCatalog.ts
|
|
21
|
+
function sample(ids, limit) {
|
|
22
|
+
return ids.length <= limit ? ids : ids.slice(0, limit);
|
|
23
|
+
}
|
|
24
|
+
async function verifyAiModelsCatalog(options) {
|
|
25
|
+
const start = Date.now();
|
|
26
|
+
const appId = _nullishCoalesce(options.appId, () => ( _chunkTF4L2NECcjs.AI_TOOLS_APP_ID));
|
|
27
|
+
const catalogId = _nullishCoalesce(options.catalogId, () => ( _chunkTF4L2NECcjs.AI_MODELS_CATALOG_ID));
|
|
28
|
+
const sampleLimit = _nullishCoalesce(options.sampleLimit, () => ( 20));
|
|
29
|
+
if (options.ensureDescriptor !== false) {
|
|
30
|
+
await _chunkDKHGWHXPcjs.ensureAiModelsCatalog.call(void 0, options.catalox, { appId, catalogId });
|
|
31
|
+
}
|
|
32
|
+
const orIds = await _asyncNullishCoalesce(options.expectedModelIds, async () => ( new Set(
|
|
33
|
+
(await _asyncNullishCoalesce(options.openRouterModels, async () => ( await new (0, _chunkDKHGWHXPcjs.OpenRouterSyncProvider)({
|
|
34
|
+
apiKey: options.openRouterApiKey,
|
|
35
|
+
query: _nullishCoalesce(options.openRouterQuery, () => ( { output_modalities: "all" }))
|
|
36
|
+
}).fetchModels()))).map((m) => m.modelId)
|
|
37
|
+
)));
|
|
38
|
+
_chunkTF4L2NECcjs.invalidateModelsCache.call(void 0, appId);
|
|
39
|
+
const client = new (0, _chunkF2F4UEFDcjs.AiModelsCatalogClient)({ catalox: options.catalox, appId, catalogId });
|
|
40
|
+
const catMap = await client.getAllModels();
|
|
41
|
+
const catIds = new Set(catMap.keys());
|
|
42
|
+
const missingInCatalox = [...orIds].filter((id) => !catIds.has(id));
|
|
43
|
+
const extraInCatalox = [...catIds].filter((id) => !orIds.has(id));
|
|
44
|
+
let supportsReasoningMissing = 0;
|
|
45
|
+
let openRouterMirrorMissing = 0;
|
|
46
|
+
for (const m of catMap.values()) {
|
|
47
|
+
if (m.supportsReasoning === void 0) supportsReasoningMissing++;
|
|
48
|
+
if (!_optionalChain([m, 'access', _ => _.openRouter, 'optionalAccess', _2 => _2.id])) openRouterMirrorMissing++;
|
|
49
|
+
}
|
|
50
|
+
const descriptor = await options.catalox.getCatalogDescriptor(
|
|
51
|
+
{ appId, superAdmin: true },
|
|
52
|
+
catalogId
|
|
53
|
+
);
|
|
54
|
+
const remoteKeys = _nullishCoalesce(_optionalChain([descriptor, 'optionalAccess', _3 => _3.queryableFields, 'optionalAccess', _4 => _4.map, 'call', _5 => _5((f) => f.key)]), () => ( []));
|
|
55
|
+
const localKeys = _chunkTF4L2NECcjs.AI_MODELS_DESCRIPTOR.queryableFields.map((f) => f.key);
|
|
56
|
+
const descriptorKeysMatch = remoteKeys.length === localKeys.length && remoteKeys.every((k, i) => k === localKeys[i]);
|
|
57
|
+
const ok = missingInCatalox.length === 0 && extraInCatalox.length === 0 && supportsReasoningMissing === 0 && openRouterMirrorMissing === 0 && descriptorKeysMatch;
|
|
58
|
+
return {
|
|
59
|
+
ok,
|
|
60
|
+
openRouterCount: orIds.size,
|
|
61
|
+
cataloxCount: catIds.size,
|
|
62
|
+
missingInCatalox: sample(missingInCatalox, sampleLimit),
|
|
63
|
+
extraInCatalox: sample(extraInCatalox, sampleLimit),
|
|
64
|
+
supportsReasoningMissing,
|
|
65
|
+
openRouterMirrorMissing,
|
|
66
|
+
descriptorKeysMatch,
|
|
67
|
+
durationMs: Date.now() - start
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// src/catalog/pruneStaleCatalogModels.ts
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
var _firebase = require('@x12i/catalox/firebase');
|
|
77
|
+
var DELETE_BATCH = 400;
|
|
78
|
+
async function pruneStaleCatalogModels(options) {
|
|
79
|
+
const { firestore, catalogId, activeModelIds, dryRun } = options;
|
|
80
|
+
const layout = await _firebase.resolveNativeItemsLayout.call(void 0, firestore, catalogId);
|
|
81
|
+
const col = layout === "legacy" ? _firebase.legacyNativeItemsCollectionRef.call(void 0, firestore, catalogId) : _firebase.flatNativeItemsCollectionRef.call(void 0, firestore, catalogId);
|
|
82
|
+
const snapshot = await col.get();
|
|
83
|
+
const toDelete = [];
|
|
84
|
+
for (const doc of snapshot.docs) {
|
|
85
|
+
const modelId = _chunkFGP3QXWLcjs.decodeModelFirestoreDocId.call(void 0, doc.id);
|
|
86
|
+
if (!activeModelIds.has(modelId)) {
|
|
87
|
+
toDelete.push(modelId);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
if (!dryRun && toDelete.length > 0) {
|
|
91
|
+
for (let i = 0; i < toDelete.length; i += DELETE_BATCH) {
|
|
92
|
+
const chunk = toDelete.slice(i, i + DELETE_BATCH);
|
|
93
|
+
const batch = firestore.batch();
|
|
94
|
+
for (const modelId of chunk) {
|
|
95
|
+
batch.delete(col.doc(_chunkFGP3QXWLcjs.encodeModelFirestoreDocId.call(void 0, modelId)));
|
|
96
|
+
}
|
|
97
|
+
await batch.commit();
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return {
|
|
101
|
+
scanned: snapshot.size,
|
|
102
|
+
pruned: toDelete.length,
|
|
103
|
+
prunedModelIds: toDelete
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// src/catalog/runAiModelsCatalogSync.ts
|
|
108
|
+
var CatalogSyncJobError = class extends Error {
|
|
109
|
+
constructor(message, sync, verify) {
|
|
110
|
+
super(message);
|
|
111
|
+
this.sync = sync;
|
|
112
|
+
this.verify = verify;
|
|
113
|
+
this.name = "CatalogSyncJobError";
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
};
|
|
118
|
+
async function runAiModelsCatalogSync(options) {
|
|
119
|
+
const verifyAfter = options.verifyAfter !== false;
|
|
120
|
+
const failOnVerifyError = options.failOnVerifyError !== false;
|
|
121
|
+
const appId = _nullishCoalesce(options.appId, () => ( _chunkTF4L2NECcjs.AI_TOOLS_APP_ID));
|
|
122
|
+
const catalogId = _nullishCoalesce(options.catalogId, () => ( _chunkTF4L2NECcjs.AI_MODELS_CATALOG_ID));
|
|
123
|
+
const ctx = { appId, superAdmin: true };
|
|
124
|
+
const sync = await _chunkDKHGWHXPcjs.syncAiModelsCatalog.call(void 0, options);
|
|
125
|
+
let prune;
|
|
126
|
+
if (options.pruneStale && !options.dryRun) {
|
|
127
|
+
prune = await pruneStaleCatalogModels({
|
|
128
|
+
firestore: options.firestore,
|
|
129
|
+
catalogId,
|
|
130
|
+
context: ctx,
|
|
131
|
+
activeModelIds: new Set(sync.syncedModelIds),
|
|
132
|
+
dryRun: options.dryRun
|
|
133
|
+
});
|
|
134
|
+
_chunkTF4L2NECcjs.invalidateModelsCache.call(void 0, appId);
|
|
135
|
+
}
|
|
136
|
+
let verify;
|
|
137
|
+
if (verifyAfter) {
|
|
138
|
+
verify = await verifyAiModelsCatalog({
|
|
139
|
+
catalox: options.catalox,
|
|
140
|
+
appId,
|
|
141
|
+
catalogId,
|
|
142
|
+
expectedModelIds: new Set(sync.syncedModelIds),
|
|
143
|
+
ensureDescriptor: false
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
const syncFailed = sync.errors.length > 0 || sync.upserted < sync.fetched;
|
|
147
|
+
const verifyFailed = verify !== void 0 && !verify.ok;
|
|
148
|
+
const ok = !syncFailed && !verifyFailed;
|
|
149
|
+
if (!ok && failOnVerifyError) {
|
|
150
|
+
const parts = [];
|
|
151
|
+
if (syncFailed) {
|
|
152
|
+
parts.push(
|
|
153
|
+
`sync incomplete: upserted ${sync.upserted}/${sync.fetched}, ${sync.errors.length} errors`
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
if (verifyFailed && verify) {
|
|
157
|
+
parts.push(
|
|
158
|
+
`verify failed: catalox ${verify.cataloxCount} vs openrouter ${verify.openRouterCount}, missing ${verify.missingInCatalox.length}, extra ${verify.extraInCatalox.length}`
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
throw new CatalogSyncJobError(parts.join("; "), sync, verify);
|
|
162
|
+
}
|
|
163
|
+
return { sync, verify, prune, ok };
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
exports.verifyAiModelsCatalog = verifyAiModelsCatalog; exports.pruneStaleCatalogModels = pruneStaleCatalogModels; exports.CatalogSyncJobError = CatalogSyncJobError; exports.runAiModelsCatalogSync = runAiModelsCatalogSync;
|
|
172
|
+
//# sourceMappingURL=chunk-HS74X2OJ.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/ami/Documents/prometheus/x12i/ai-tools/dist/chunk-HS74X2OJ.cjs","../src/catalog/verifyAiModelsCatalog.ts","../src/catalog/pruneStaleCatalogModels.ts","../src/catalog/runAiModelsCatalogSync.ts"],"names":[],"mappings":"AAAA;AACE;AACA;AACA;AACF,wDAA6B;AAC7B;AACE;AACA;AACF,wDAA6B;AAC7B;AACE;AACF,wDAA6B;AAC7B;AACE;AACA;AACA;AACA;AACF,wDAA6B;AAC7B;AACA;ACsBA,SAAS,MAAA,CAAO,GAAA,EAAe,KAAA,EAAyB;AACtD,EAAA,OAAO,GAAA,CAAI,OAAA,GAAU,MAAA,EAAQ,IAAA,EAAM,GAAA,CAAI,KAAA,CAAM,CAAA,EAAG,KAAK,CAAA;AACvD;AAMA,MAAA,SAAsB,qBAAA,CACpB,OAAA,EAC8B;AAC9B,EAAA,MAAM,MAAA,EAAQ,IAAA,CAAK,GAAA,CAAI,CAAA;AACvB,EAAA,MAAM,MAAA,mBAAQ,OAAA,CAAQ,KAAA,UAAS,mCAAA;AAC/B,EAAA,MAAM,UAAA,mBAAY,OAAA,CAAQ,SAAA,UAAa,wCAAA;AACvC,EAAA,MAAM,YAAA,mBAAc,OAAA,CAAQ,WAAA,UAAe,IAAA;AAE3C,EAAA,GAAA,CAAI,OAAA,CAAQ,iBAAA,IAAqB,KAAA,EAAO;AACtC,IAAA,MAAM,qDAAA,OAAsB,CAAQ,OAAA,EAAS,EAAE,KAAA,EAAO,UAAU,CAAC,CAAA;AAAA,EACnE;AAEA,EAAA,MAAM,MAAA,8BACJ,OAAA,CAAQ,gBAAA,gBACR,IAAI,GAAA;AAAA,IAAA,6BAEA,OAAA,CAAQ,gBAAA,gBACP,MAAM,IAAI,6CAAA,CAAuB;AAAA,MAChC,MAAA,EAAQ,OAAA,CAAQ,gBAAA;AAAA,MAChB,KAAA,mBAAO,OAAA,CAAQ,eAAA,UAAmB,EAAE,iBAAA,EAAmB,MAAM;AAAA,IAC/D,CAAC,CAAA,CAAE,WAAA,CAAY,GAAA,CAAA,CACf,GAAA,CAAI,CAAC,CAAA,EAAA,GAAM,CAAA,CAAE,OAAO;AAAA,EACxB,GAAA;AAEF,EAAA,qDAAA,KAA2B,CAAA;AAC3B,EAAA,MAAM,OAAA,EAAS,IAAI,4CAAA,CAAsB,EAAE,OAAA,EAAS,OAAA,CAAQ,OAAA,EAAS,KAAA,EAAO,UAAU,CAAC,CAAA;AACvF,EAAA,MAAM,OAAA,EAAS,MAAM,MAAA,CAAO,YAAA,CAAa,CAAA;AACzC,EAAA,MAAM,OAAA,EAAS,IAAI,GAAA,CAAI,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA;AAEpC,EAAA,MAAM,iBAAA,EAAmB,CAAC,GAAG,KAAK,CAAA,CAAE,MAAA,CAAO,CAAC,EAAA,EAAA,GAAO,CAAC,MAAA,CAAO,GAAA,CAAI,EAAE,CAAC,CAAA;AAClE,EAAA,MAAM,eAAA,EAAiB,CAAC,GAAG,MAAM,CAAA,CAAE,MAAA,CAAO,CAAC,EAAA,EAAA,GAAO,CAAC,KAAA,CAAM,GAAA,CAAI,EAAE,CAAC,CAAA;AAEhE,EAAA,IAAI,yBAAA,EAA2B,CAAA;AAC/B,EAAA,IAAI,wBAAA,EAA0B,CAAA;AAC9B,EAAA,IAAA,CAAA,MAAW,EAAA,GAAK,MAAA,CAAO,MAAA,CAAO,CAAA,EAAG;AAC/B,IAAA,GAAA,CAAI,CAAA,CAAE,kBAAA,IAAsB,KAAA,CAAA,EAAW,wBAAA,EAAA;AACvC,IAAA,GAAA,CAAI,iBAAC,CAAA,mBAAE,UAAA,6BAAY,IAAA,EAAI,uBAAA,EAAA;AAAA,EACzB;AAEA,EAAA,MAAM,WAAA,EAAa,MAAM,OAAA,CAAQ,OAAA,CAAQ,oBAAA;AAAA,IACvC,EAAE,KAAA,EAAO,UAAA,EAAY,KAAK,CAAA;AAAA,IAC1B;AAAA,EACF,CAAA;AACA,EAAA,MAAM,WAAA,mCAAa,UAAA,6BAAY,eAAA,6BAAiB,GAAA,mBAAI,CAAC,CAAA,EAAA,GAAM,CAAA,CAAE,GAAG,GAAA,UAAK,CAAC,GAAA;AACtE,EAAA,MAAM,UAAA,EAAY,sCAAA,CAAqB,eAAA,CAAgB,GAAA,CAAI,CAAC,CAAA,EAAA,GAAM,CAAA,CAAE,GAAG,CAAA;AACvE,EAAA,MAAM,oBAAA,EACJ,UAAA,CAAW,OAAA,IAAW,SAAA,CAAU,OAAA,GAChC,UAAA,CAAW,KAAA,CAAM,CAAC,CAAA,EAAG,CAAA,EAAA,GAAM,EAAA,IAAM,SAAA,CAAU,CAAC,CAAC,CAAA;AAE/C,EAAA,MAAM,GAAA,EACJ,gBAAA,CAAiB,OAAA,IAAW,EAAA,GAC5B,cAAA,CAAe,OAAA,IAAW,EAAA,GAC1B,yBAAA,IAA6B,EAAA,GAC7B,wBAAA,IAA4B,EAAA,GAC5B,mBAAA;AAEF,EAAA,OAAO;AAAA,IACL,EAAA;AAAA,IACA,eAAA,EAAiB,KAAA,CAAM,IAAA;AAAA,IACvB,YAAA,EAAc,MAAA,CAAO,IAAA;AAAA,IACrB,gBAAA,EAAkB,MAAA,CAAO,gBAAA,EAAkB,WAAW,CAAA;AAAA,IACtD,cAAA,EAAgB,MAAA,CAAO,cAAA,EAAgB,WAAW,CAAA;AAAA,IAClD,wBAAA;AAAA,IACA,uBAAA;AAAA,IACA,mBAAA;AAAA,IACA,UAAA,EAAY,IAAA,CAAK,GAAA,CAAI,EAAA,EAAI;AAAA,EAC3B,CAAA;AACF;AD/CA;AACA;AEpEA;AACE;AACA;AACA;AAAA,kDACK;AAkBP,IAAM,aAAA,EAAe,GAAA;AAMrB,MAAA,SAAsB,uBAAA,CACpB,OAAA,EACwC;AACxC,EAAA,MAAM,EAAE,SAAA,EAAW,SAAA,EAAW,cAAA,EAAgB,OAAO,EAAA,EAAI,OAAA;AACzD,EAAA,MAAM,OAAA,EAAS,MAAM,gDAAA,SAAyB,EAAW,SAAS,CAAA;AAClE,EAAA,MAAM,IAAA,EACJ,OAAA,IAAW,SAAA,EACP,sDAAA,SAA+B,EAAW,SAAS,EAAA,EACnD,oDAAA,SAA6B,EAAW,SAAS,CAAA;AAEvD,EAAA,MAAM,SAAA,EAAW,MAAM,GAAA,CAAI,GAAA,CAAI,CAAA;AAC/B,EAAA,MAAM,SAAA,EAAqB,CAAC,CAAA;AAE5B,EAAA,IAAA,CAAA,MAAW,IAAA,GAAO,QAAA,CAAS,IAAA,EAAM;AAC/B,IAAA,MAAM,QAAA,EAAU,yDAAA,GAA0B,CAAI,EAAE,CAAA;AAChD,IAAA,GAAA,CAAI,CAAC,cAAA,CAAe,GAAA,CAAI,OAAO,CAAA,EAAG;AAChC,MAAA,QAAA,CAAS,IAAA,CAAK,OAAO,CAAA;AAAA,IACvB;AAAA,EACF;AAEA,EAAA,GAAA,CAAI,CAAC,OAAA,GAAU,QAAA,CAAS,OAAA,EAAS,CAAA,EAAG;AAClC,IAAA,IAAA,CAAA,IAAS,EAAA,EAAI,CAAA,EAAG,EAAA,EAAI,QAAA,CAAS,MAAA,EAAQ,EAAA,GAAK,YAAA,EAAc;AACtD,MAAA,MAAM,MAAA,EAAQ,QAAA,CAAS,KAAA,CAAM,CAAA,EAAG,EAAA,EAAI,YAAY,CAAA;AAChD,MAAA,MAAM,MAAA,EAAQ,SAAA,CAAU,KAAA,CAAM,CAAA;AAC9B,MAAA,IAAA,CAAA,MAAW,QAAA,GAAW,KAAA,EAAO;AAC3B,QAAA,KAAA,CAAM,MAAA,CAAO,GAAA,CAAI,GAAA,CAAI,yDAAA,OAAiC,CAAC,CAAC,CAAA;AAAA,MAC1D;AACA,MAAA,MAAM,KAAA,CAAM,MAAA,CAAO,CAAA;AAAA,IACrB;AAAA,EACF;AAEA,EAAA,OAAO;AAAA,IACL,OAAA,EAAS,QAAA,CAAS,IAAA;AAAA,IAClB,MAAA,EAAQ,QAAA,CAAS,MAAA;AAAA,IACjB,cAAA,EAAgB;AAAA,EAClB,CAAA;AACF;AFuCA;AACA;AG3EO,IAAM,oBAAA,EAAN,MAAA,QAAkC,MAAM;AAAA,EAC7C,WAAA,CACE,OAAA,EACgB,IAAA,EACA,MAAA,EAChB;AACA,IAAA,KAAA,CAAM,OAAO,CAAA;AAHG,IAAA,IAAA,CAAA,KAAA,EAAA,IAAA;AACA,IAAA,IAAA,CAAA,OAAA,EAAA,MAAA;AAGhB,IAAA,IAAA,CAAK,KAAA,EAAO,qBAAA;AAAA,EACd;AAAA,EALkB;AAAA,EACA;AAKpB,CAAA;AAMA,MAAA,SAAsB,sBAAA,CACpB,OAAA,EAC+B;AAC/B,EAAA,MAAM,YAAA,EAAc,OAAA,CAAQ,YAAA,IAAgB,KAAA;AAC5C,EAAA,MAAM,kBAAA,EAAoB,OAAA,CAAQ,kBAAA,IAAsB,KAAA;AACxD,EAAA,MAAM,MAAA,mBAAQ,OAAA,CAAQ,KAAA,UAAS,mCAAA;AAC/B,EAAA,MAAM,UAAA,mBAAY,OAAA,CAAQ,SAAA,UAAa,wCAAA;AACvC,EAAA,MAAM,IAAA,EAAM,EAAE,KAAA,EAAO,UAAA,EAAY,KAAc,CAAA;AAE/C,EAAA,MAAM,KAAA,EAAO,MAAM,mDAAA,OAA2B,CAAA;AAE9C,EAAA,IAAI,KAAA;AAEJ,EAAA,GAAA,CAAI,OAAA,CAAQ,WAAA,GAAc,CAAC,OAAA,CAAQ,MAAA,EAAQ;AACzC,IAAA,MAAA,EAAQ,MAAM,uBAAA,CAAwB;AAAA,MACpC,SAAA,EAAW,OAAA,CAAQ,SAAA;AAAA,MACnB,SAAA;AAAA,MACA,OAAA,EAAS,GAAA;AAAA,MACT,cAAA,EAAgB,IAAI,GAAA,CAAI,IAAA,CAAK,cAAc,CAAA;AAAA,MAC3C,MAAA,EAAQ,OAAA,CAAQ;AAAA,IAClB,CAAC,CAAA;AACD,IAAA,qDAAA,KAA2B,CAAA;AAAA,EAC7B;AAEA,EAAA,IAAI,MAAA;AACJ,EAAA,GAAA,CAAI,WAAA,EAAa;AACf,IAAA,OAAA,EAAS,MAAM,qBAAA,CAAsB;AAAA,MACnC,OAAA,EAAS,OAAA,CAAQ,OAAA;AAAA,MACjB,KAAA;AAAA,MACA,SAAA;AAAA,MACA,gBAAA,EAAkB,IAAI,GAAA,CAAI,IAAA,CAAK,cAAc,CAAA;AAAA,MAC7C,gBAAA,EAAkB;AAAA,IACpB,CAAC,CAAA;AAAA,EACH;AAEA,EAAA,MAAM,WAAA,EAAa,IAAA,CAAK,MAAA,CAAO,OAAA,EAAS,EAAA,GAAK,IAAA,CAAK,SAAA,EAAW,IAAA,CAAK,OAAA;AAClE,EAAA,MAAM,aAAA,EAAe,OAAA,IAAW,KAAA,EAAA,GAAa,CAAC,MAAA,CAAO,EAAA;AACrD,EAAA,MAAM,GAAA,EAAK,CAAC,WAAA,GAAc,CAAC,YAAA;AAE3B,EAAA,GAAA,CAAI,CAAC,GAAA,GAAM,iBAAA,EAAmB;AAC5B,IAAA,MAAM,MAAA,EAAkB,CAAC,CAAA;AACzB,IAAA,GAAA,CAAI,UAAA,EAAY;AACd,MAAA,KAAA,CAAM,IAAA;AAAA,QACJ,CAAA,0BAAA,EAA6B,IAAA,CAAK,QAAQ,CAAA,CAAA,EAAI,IAAA,CAAK,OAAO,CAAA,EAAA,EAAK,IAAA,CAAK,MAAA,CAAO,MAAM,CAAA,OAAA;AAAA,MACnF,CAAA;AAAA,IACF;AACA,IAAA,GAAA,CAAI,aAAA,GAAgB,MAAA,EAAQ;AAC1B,MAAA,KAAA,CAAM,IAAA;AAAA,QACJ,CAAA,uBAAA,EAA0B,MAAA,CAAO,YAAY,CAAA,eAAA,EAAkB,MAAA,CAAO,eAAe,CAAA,UAAA,EACxE,MAAA,CAAO,gBAAA,CAAiB,MAAM,CAAA,QAAA,EAAW,MAAA,CAAO,cAAA,CAAe,MAAM,CAAA;AAAA,MAAA;AACpF,IAAA;AAEF,IAAA;AAA4D,EAAA;AAG9D,EAAA;AACF;AH8DA;AACA;AACA;AACA;AACA;AACA;AACA","file":"/Users/ami/Documents/prometheus/x12i/ai-tools/dist/chunk-HS74X2OJ.cjs","sourcesContent":[null,"import type { Catalox } from \"@x12i/catalox\";\nimport { invalidateModelsCache } from \"../cache/modelCache.js\";\nimport { AiModelsCatalogClient } from \"../catalox/AiModelsCatalogClient.js\";\nimport type { AiModelRecord } from \"../models/types.js\";\nimport { OpenRouterSyncProvider } from \"../sync/OpenRouterSyncProvider.js\";\nimport type { OpenRouterModelsQuery } from \"../models/openrouter.types.js\";\nimport {\n AI_MODELS_CATALOG_ID,\n AI_MODELS_DESCRIPTOR,\n AI_TOOLS_APP_ID,\n} from \"./aiModelsCatalogDescriptor.js\";\nimport { ensureAiModelsCatalog } from \"./ensureAiModelsCatalog.js\";\n\nexport type CatalogVerifyOptions = {\n catalox: Catalox;\n appId?: string;\n catalogId?: string;\n /** When set, compare against this id set (e.g. from a sync that just ran). */\n expectedModelIds?: Set<string>;\n /** When set, compare against this list instead of fetching OpenRouter. */\n openRouterModels?: AiModelRecord[];\n openRouterApiKey?: string;\n openRouterQuery?: OpenRouterModelsQuery;\n /** Ensure descriptor is registered before reading (default true). */\n ensureDescriptor?: boolean;\n /** Max model ids to include in report samples (default 20). */\n sampleLimit?: number;\n};\n\nexport type CatalogVerifyReport = {\n ok: boolean;\n openRouterCount: number;\n cataloxCount: number;\n missingInCatalox: string[];\n extraInCatalox: string[];\n supportsReasoningMissing: number;\n openRouterMirrorMissing: number;\n descriptorKeysMatch: boolean;\n durationMs: number;\n};\n\nfunction sample(ids: string[], limit: number): string[] {\n return ids.length <= limit ? ids : ids.slice(0, limit);\n}\n\n/**\n * Compare the live OpenRouter catalog with Catalox/Firestore ai-models items.\n * Use after sync in CI/cron, or standalone health checks.\n */\nexport async function verifyAiModelsCatalog(\n options: CatalogVerifyOptions,\n): Promise<CatalogVerifyReport> {\n const start = Date.now();\n const appId = options.appId ?? AI_TOOLS_APP_ID;\n const catalogId = options.catalogId ?? AI_MODELS_CATALOG_ID;\n const sampleLimit = options.sampleLimit ?? 20;\n\n if (options.ensureDescriptor !== false) {\n await ensureAiModelsCatalog(options.catalox, { appId, catalogId });\n }\n\n const orIds =\n options.expectedModelIds ??\n new Set(\n (\n options.openRouterModels ??\n (await new OpenRouterSyncProvider({\n apiKey: options.openRouterApiKey,\n query: options.openRouterQuery ?? { output_modalities: \"all\" },\n }).fetchModels())\n ).map((m) => m.modelId),\n );\n\n invalidateModelsCache(appId);\n const client = new AiModelsCatalogClient({ catalox: options.catalox, appId, catalogId });\n const catMap = await client.getAllModels();\n const catIds = new Set(catMap.keys());\n\n const missingInCatalox = [...orIds].filter((id) => !catIds.has(id));\n const extraInCatalox = [...catIds].filter((id) => !orIds.has(id));\n\n let supportsReasoningMissing = 0;\n let openRouterMirrorMissing = 0;\n for (const m of catMap.values()) {\n if (m.supportsReasoning === undefined) supportsReasoningMissing++;\n if (!m.openRouter?.id) openRouterMirrorMissing++;\n }\n\n const descriptor = await options.catalox.getCatalogDescriptor(\n { appId, superAdmin: true },\n catalogId,\n );\n const remoteKeys = descriptor?.queryableFields?.map((f) => f.key) ?? [];\n const localKeys = AI_MODELS_DESCRIPTOR.queryableFields.map((f) => f.key);\n const descriptorKeysMatch =\n remoteKeys.length === localKeys.length &&\n remoteKeys.every((k, i) => k === localKeys[i]);\n\n const ok =\n missingInCatalox.length === 0 &&\n extraInCatalox.length === 0 &&\n supportsReasoningMissing === 0 &&\n openRouterMirrorMissing === 0 &&\n descriptorKeysMatch;\n\n return {\n ok,\n openRouterCount: orIds.size,\n cataloxCount: catIds.size,\n missingInCatalox: sample(missingInCatalox, sampleLimit),\n extraInCatalox: sample(extraInCatalox, sampleLimit),\n supportsReasoningMissing,\n openRouterMirrorMissing,\n descriptorKeysMatch,\n durationMs: Date.now() - start,\n };\n}\n","import type { Firestore } from \"firebase-admin/firestore\";\nimport type { CataloxContext } from \"@x12i/catalox\";\nimport {\n flatNativeItemsCollectionRef,\n legacyNativeItemsCollectionRef,\n resolveNativeItemsLayout,\n} from \"@x12i/catalox/firebase\";\nimport { decodeModelFirestoreDocId, encodeModelFirestoreDocId } from \"../catalox/modelDocId.js\";\n\nexport type PruneStaleCatalogModelsOptions = {\n firestore: Firestore;\n catalogId: string;\n context: CataloxContext;\n /** Canonical model ids from the latest OpenRouter fetch. */\n activeModelIds: Set<string>;\n dryRun?: boolean;\n};\n\nexport type PruneStaleCatalogModelsResult = {\n scanned: number;\n pruned: number;\n prunedModelIds: string[];\n};\n\nconst DELETE_BATCH = 400;\n\n/**\n * Remove catalog documents that are no longer listed on OpenRouter.\n * Off by default — enable explicitly when running production sync jobs.\n */\nexport async function pruneStaleCatalogModels(\n options: PruneStaleCatalogModelsOptions,\n): Promise<PruneStaleCatalogModelsResult> {\n const { firestore, catalogId, activeModelIds, dryRun } = options;\n const layout = await resolveNativeItemsLayout(firestore, catalogId);\n const col =\n layout === \"legacy\"\n ? legacyNativeItemsCollectionRef(firestore, catalogId)\n : flatNativeItemsCollectionRef(firestore, catalogId);\n\n const snapshot = await col.get();\n const toDelete: string[] = [];\n\n for (const doc of snapshot.docs) {\n const modelId = decodeModelFirestoreDocId(doc.id);\n if (!activeModelIds.has(modelId)) {\n toDelete.push(modelId);\n }\n }\n\n if (!dryRun && toDelete.length > 0) {\n for (let i = 0; i < toDelete.length; i += DELETE_BATCH) {\n const chunk = toDelete.slice(i, i + DELETE_BATCH);\n const batch = firestore.batch();\n for (const modelId of chunk) {\n batch.delete(col.doc(encodeModelFirestoreDocId(modelId)));\n }\n await batch.commit();\n }\n }\n\n return {\n scanned: snapshot.size,\n pruned: toDelete.length,\n prunedModelIds: toDelete,\n };\n}\n","import { invalidateModelsCache } from \"../cache/modelCache.js\";\nimport { syncAiModelsCatalog } from \"../sync/syncAiModelsCatalog.js\";\nimport type { SyncOptions, SyncResult } from \"../sync/syncAiModelsCatalog.js\";\nimport { pruneStaleCatalogModels } from \"./pruneStaleCatalogModels.js\";\nimport type { CatalogVerifyReport } from \"./verifyAiModelsCatalog.js\";\nimport { verifyAiModelsCatalog } from \"./verifyAiModelsCatalog.js\";\nimport { AI_MODELS_CATALOG_ID, AI_TOOLS_APP_ID } from \"./aiModelsCatalogDescriptor.js\";\n\nexport type CatalogSyncJobOptions = SyncOptions & {\n /**\n * After upsert, compare Catalox to OpenRouter (default true).\n * Set false only when you will verify separately.\n */\n verifyAfter?: boolean;\n /** Fail the job when verification does not pass (default true). */\n failOnVerifyError?: boolean;\n /** Delete Firestore rows not present in the latest OpenRouter list (default false). */\n pruneStale?: boolean;\n};\n\nexport type CatalogSyncJobResult = {\n sync: SyncResult;\n verify?: CatalogVerifyReport;\n prune?: {\n scanned: number;\n pruned: number;\n prunedModelIds: string[];\n };\n ok: boolean;\n};\n\nexport class CatalogSyncJobError extends Error {\n constructor(\n message: string,\n public readonly sync: SyncResult,\n public readonly verify?: CatalogVerifyReport,\n ) {\n super(message);\n this.name = \"CatalogSyncJobError\";\n }\n}\n\n/**\n * Production entry point: sync OpenRouter → Catalox, optionally prune stale rows, then verify.\n * Suitable for cron, Cloud Run jobs, and `ai-tools sync`.\n */\nexport async function runAiModelsCatalogSync(\n options: CatalogSyncJobOptions,\n): Promise<CatalogSyncJobResult> {\n const verifyAfter = options.verifyAfter !== false;\n const failOnVerifyError = options.failOnVerifyError !== false;\n const appId = options.appId ?? AI_TOOLS_APP_ID;\n const catalogId = options.catalogId ?? AI_MODELS_CATALOG_ID;\n const ctx = { appId, superAdmin: true as const };\n\n const sync = await syncAiModelsCatalog(options);\n\n let prune: CatalogSyncJobResult[\"prune\"];\n\n if (options.pruneStale && !options.dryRun) {\n prune = await pruneStaleCatalogModels({\n firestore: options.firestore,\n catalogId,\n context: ctx,\n activeModelIds: new Set(sync.syncedModelIds),\n dryRun: options.dryRun,\n });\n invalidateModelsCache(appId);\n }\n\n let verify: CatalogVerifyReport | undefined;\n if (verifyAfter) {\n verify = await verifyAiModelsCatalog({\n catalox: options.catalox,\n appId,\n catalogId,\n expectedModelIds: new Set(sync.syncedModelIds),\n ensureDescriptor: false,\n });\n }\n\n const syncFailed = sync.errors.length > 0 || sync.upserted < sync.fetched;\n const verifyFailed = verify !== undefined && !verify.ok;\n const ok = !syncFailed && !verifyFailed;\n\n if (!ok && failOnVerifyError) {\n const parts: string[] = [];\n if (syncFailed) {\n parts.push(\n `sync incomplete: upserted ${sync.upserted}/${sync.fetched}, ${sync.errors.length} errors`,\n );\n }\n if (verifyFailed && verify) {\n parts.push(\n `verify failed: catalox ${verify.cataloxCount} vs openrouter ${verify.openRouterCount}, ` +\n `missing ${verify.missingInCatalox.length}, extra ${verify.extraInCatalox.length}`,\n );\n }\n throw new CatalogSyncJobError(parts.join(\"; \"), sync, verify);\n }\n\n return { sync, verify, prune, ok };\n}\n"]}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
// src/catalox/modelDocId.ts
|
|
2
|
+
var CX = "cx~";
|
|
3
|
+
var MODEL_MARKER = "m~";
|
|
4
|
+
function encodeModelFirestoreDocId(modelId) {
|
|
5
|
+
if (!modelId.includes("/")) return modelId;
|
|
6
|
+
return `${CX}${MODEL_MARKER}${Buffer.from(modelId, "utf8").toString("base64url")}`;
|
|
7
|
+
}
|
|
8
|
+
function decodeModelFirestoreDocId(docId) {
|
|
9
|
+
const prefix = `${CX}${MODEL_MARKER}`;
|
|
10
|
+
if (docId.startsWith(prefix)) {
|
|
11
|
+
return Buffer.from(docId.slice(prefix.length), "base64url").toString("utf8");
|
|
12
|
+
}
|
|
13
|
+
return docId;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
// src/catalox/upsertAiModelRecord.ts
|
|
17
|
+
import {
|
|
18
|
+
flatNativeItemsCollectionRef,
|
|
19
|
+
legacyNativeItemsCollectionRef,
|
|
20
|
+
resolveNativeItemsLayout
|
|
21
|
+
} from "@x12i/catalox/firebase";
|
|
22
|
+
function sanitizeForFirestore(value) {
|
|
23
|
+
return JSON.parse(JSON.stringify(value));
|
|
24
|
+
}
|
|
25
|
+
function buildIndexed(record) {
|
|
26
|
+
return {
|
|
27
|
+
modelId: record.modelId,
|
|
28
|
+
providerId: record.providerId,
|
|
29
|
+
name: record.name,
|
|
30
|
+
canonicalSlug: record.canonicalSlug,
|
|
31
|
+
contextLength: record.contextLength,
|
|
32
|
+
status: record.status,
|
|
33
|
+
primaryOutputModality: record.primaryOutputModality,
|
|
34
|
+
supportsTools: record.supportsTools,
|
|
35
|
+
supportsReasoning: record.supportsReasoning,
|
|
36
|
+
supportsStreaming: record.supportsStreaming,
|
|
37
|
+
isModerated: record.isModerated,
|
|
38
|
+
syncedAt: record.syncedAt
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
function buildCatalogItemUpsertPayload(record, existing, options) {
|
|
42
|
+
const { catalogId, appId, now } = options;
|
|
43
|
+
return sanitizeForFirestore({
|
|
44
|
+
itemId: record.modelId,
|
|
45
|
+
catalogId,
|
|
46
|
+
appScopedOwnerId: appId,
|
|
47
|
+
indexed: buildIndexed(record),
|
|
48
|
+
data: sanitizeForFirestore(record),
|
|
49
|
+
metadata: {
|
|
50
|
+
createdAt: String(existing?.metadata?.createdAt ?? now),
|
|
51
|
+
lastUpdate: now,
|
|
52
|
+
domainIds: [],
|
|
53
|
+
agentIds: []
|
|
54
|
+
},
|
|
55
|
+
version: (existing?.version ?? 0) + 1
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
var EXISTING_READ_BATCH = 50;
|
|
59
|
+
async function loadExistingByStorageId(col, records) {
|
|
60
|
+
const map = /* @__PURE__ */ new Map();
|
|
61
|
+
for (let i = 0; i < records.length; i += EXISTING_READ_BATCH) {
|
|
62
|
+
const slice = records.slice(i, i + EXISTING_READ_BATCH);
|
|
63
|
+
const snaps = await Promise.all(
|
|
64
|
+
slice.map((r) => col.doc(encodeModelFirestoreDocId(r.modelId)).get())
|
|
65
|
+
);
|
|
66
|
+
for (let j = 0; j < slice.length; j++) {
|
|
67
|
+
const storageDocId = encodeModelFirestoreDocId(slice[j].modelId);
|
|
68
|
+
const data = snaps[j].data();
|
|
69
|
+
if (data) map.set(storageDocId, data);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return map;
|
|
73
|
+
}
|
|
74
|
+
async function upsertAiModelRecord(options) {
|
|
75
|
+
const { record, catalogId, context, firestore } = options;
|
|
76
|
+
const appId = context.appId ?? "ai-tools";
|
|
77
|
+
const layout = await resolveNativeItemsLayout(firestore, catalogId);
|
|
78
|
+
const col = layout === "legacy" ? legacyNativeItemsCollectionRef(firestore, catalogId) : flatNativeItemsCollectionRef(firestore, catalogId);
|
|
79
|
+
const storageDocId = encodeModelFirestoreDocId(record.modelId);
|
|
80
|
+
const existing = (await col.doc(storageDocId).get()).data();
|
|
81
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
82
|
+
const payload = buildCatalogItemUpsertPayload(record, existing, { catalogId, appId, now });
|
|
83
|
+
await col.doc(storageDocId).set(payload, { merge: true });
|
|
84
|
+
}
|
|
85
|
+
var BATCH_SIZE = 400;
|
|
86
|
+
var DEFAULT_CHUNK_RETRIES = 3;
|
|
87
|
+
var RETRY_BASE_MS = 500;
|
|
88
|
+
function sleep(ms) {
|
|
89
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
90
|
+
}
|
|
91
|
+
async function commitChunkWithRetry(firestore, col, chunk, existingByStorageId, options) {
|
|
92
|
+
const { catalogId, appId, now, maxChunkRetries } = options;
|
|
93
|
+
for (let attempt = 0; attempt < maxChunkRetries; attempt++) {
|
|
94
|
+
try {
|
|
95
|
+
const batch = firestore.batch();
|
|
96
|
+
for (const record of chunk) {
|
|
97
|
+
const storageDocId = encodeModelFirestoreDocId(record.modelId);
|
|
98
|
+
const existing = existingByStorageId.get(storageDocId);
|
|
99
|
+
const payload = buildCatalogItemUpsertPayload(record, existing, { catalogId, appId, now });
|
|
100
|
+
batch.set(col.doc(storageDocId), payload, { merge: true });
|
|
101
|
+
}
|
|
102
|
+
await batch.commit();
|
|
103
|
+
return { upserted: chunk.length, failed: [] };
|
|
104
|
+
} catch {
|
|
105
|
+
if (attempt < maxChunkRetries - 1) {
|
|
106
|
+
await sleep(RETRY_BASE_MS * 2 ** attempt);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
const failed = [];
|
|
111
|
+
let upserted = 0;
|
|
112
|
+
for (const record of chunk) {
|
|
113
|
+
try {
|
|
114
|
+
const storageDocId = encodeModelFirestoreDocId(record.modelId);
|
|
115
|
+
const existing = existingByStorageId.get(storageDocId);
|
|
116
|
+
const payload = buildCatalogItemUpsertPayload(record, existing, { catalogId, appId, now });
|
|
117
|
+
await col.doc(storageDocId).set(payload, { merge: true });
|
|
118
|
+
upserted++;
|
|
119
|
+
} catch (error) {
|
|
120
|
+
failed.push({
|
|
121
|
+
modelId: record.modelId,
|
|
122
|
+
error: error instanceof Error ? error.message : String(error)
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return { upserted, failed };
|
|
127
|
+
}
|
|
128
|
+
async function batchUpsertAiModelRecords(firestore, catalogId, context, records, batchOptions = {}) {
|
|
129
|
+
const layout = await resolveNativeItemsLayout(firestore, catalogId);
|
|
130
|
+
const col = layout === "legacy" ? legacyNativeItemsCollectionRef(firestore, catalogId) : flatNativeItemsCollectionRef(firestore, catalogId);
|
|
131
|
+
const appId = context.appId ?? "ai-tools";
|
|
132
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
133
|
+
const maxChunkRetries = batchOptions.maxChunkRetries ?? DEFAULT_CHUNK_RETRIES;
|
|
134
|
+
const total = records.length;
|
|
135
|
+
batchOptions.onProgress?.({ completed: 0, total, phase: "loading-existing" });
|
|
136
|
+
const existingByStorageId = await loadExistingByStorageId(col, records);
|
|
137
|
+
let completed = 0;
|
|
138
|
+
let upserted = 0;
|
|
139
|
+
const failed = [];
|
|
140
|
+
for (let i = 0; i < records.length; i += BATCH_SIZE) {
|
|
141
|
+
const chunk = records.slice(i, i + BATCH_SIZE);
|
|
142
|
+
const chunkResult = await commitChunkWithRetry(firestore, col, chunk, existingByStorageId, {
|
|
143
|
+
catalogId,
|
|
144
|
+
appId,
|
|
145
|
+
now,
|
|
146
|
+
maxChunkRetries
|
|
147
|
+
});
|
|
148
|
+
upserted += chunkResult.upserted;
|
|
149
|
+
failed.push(...chunkResult.failed);
|
|
150
|
+
completed += chunk.length;
|
|
151
|
+
batchOptions.onProgress?.({ completed, total, phase: "writing" });
|
|
152
|
+
}
|
|
153
|
+
return { upserted, failed };
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export {
|
|
157
|
+
encodeModelFirestoreDocId,
|
|
158
|
+
decodeModelFirestoreDocId,
|
|
159
|
+
sanitizeForFirestore,
|
|
160
|
+
upsertAiModelRecord,
|
|
161
|
+
batchUpsertAiModelRecords
|
|
162
|
+
};
|
|
163
|
+
//# sourceMappingURL=chunk-HYGXZY25.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/catalox/modelDocId.ts","../src/catalox/upsertAiModelRecord.ts"],"sourcesContent":["/**\n * Firestore document ids cannot contain `/`. OpenRouter model ids use `provider/model`.\n * We encode global item doc ids as `cx~m~<base64url(modelId)>` while keeping logical `modelId` in data.\n */\nconst CX = \"cx~\";\nconst MODEL_MARKER = \"m~\";\n\nexport function encodeModelFirestoreDocId(modelId: string): string {\n if (!modelId.includes(\"/\")) return modelId;\n return `${CX}${MODEL_MARKER}${Buffer.from(modelId, \"utf8\").toString(\"base64url\")}`;\n}\n\nexport function decodeModelFirestoreDocId(docId: string): string {\n const prefix = `${CX}${MODEL_MARKER}`;\n if (docId.startsWith(prefix)) {\n return Buffer.from(docId.slice(prefix.length), \"base64url\").toString(\"utf8\");\n }\n return docId;\n}\n\nexport function modelIdNeedsEncodedDocId(modelId: string): boolean {\n return modelId.includes(\"/\");\n}\n","import type { Firestore } from \"firebase-admin/firestore\";\nimport type { CataloxContext } from \"@x12i/catalox\";\nimport {\n flatNativeItemsCollectionRef,\n legacyNativeItemsCollectionRef,\n resolveNativeItemsLayout,\n} from \"@x12i/catalox/firebase\";\nimport type { AiModelRecord } from \"../models/types.js\";\nimport { encodeModelFirestoreDocId } from \"./modelDocId.js\";\n\n/** Firestore rejects `undefined` — strip before write. */\nexport function sanitizeForFirestore<T>(value: T): T {\n return JSON.parse(JSON.stringify(value)) as T;\n}\n\nfunction buildIndexed(record: AiModelRecord): Record<string, string | number | boolean> {\n return {\n modelId: record.modelId,\n providerId: record.providerId,\n name: record.name,\n canonicalSlug: record.canonicalSlug,\n contextLength: record.contextLength,\n status: record.status,\n primaryOutputModality: record.primaryOutputModality,\n supportsTools: record.supportsTools,\n supportsReasoning: record.supportsReasoning,\n supportsStreaming: record.supportsStreaming,\n isModerated: record.isModerated,\n syncedAt: record.syncedAt,\n };\n}\n\ntype ExistingCatalogDoc = {\n metadata?: { createdAt?: string };\n version?: number;\n};\n\n/** @internal Exported for unit tests. */\nexport function buildCatalogItemUpsertPayload(\n record: AiModelRecord,\n existing: ExistingCatalogDoc | undefined,\n options: { catalogId: string; appId: string; now: string },\n) {\n const { catalogId, appId, now } = options;\n return sanitizeForFirestore({\n itemId: record.modelId,\n catalogId,\n appScopedOwnerId: appId,\n indexed: buildIndexed(record),\n data: sanitizeForFirestore(record),\n metadata: {\n createdAt: String(existing?.metadata?.createdAt ?? now),\n lastUpdate: now,\n domainIds: [],\n agentIds: [],\n },\n version: (existing?.version ?? 0) + 1,\n });\n}\n\nconst EXISTING_READ_BATCH = 50;\n\nasync function loadExistingByStorageId(\n col: ReturnType<typeof flatNativeItemsCollectionRef>,\n records: AiModelRecord[],\n): Promise<Map<string, ExistingCatalogDoc>> {\n const map = new Map<string, ExistingCatalogDoc>();\n for (let i = 0; i < records.length; i += EXISTING_READ_BATCH) {\n const slice = records.slice(i, i + EXISTING_READ_BATCH);\n const snaps = await Promise.all(\n slice.map((r) => col.doc(encodeModelFirestoreDocId(r.modelId)).get()),\n );\n for (let j = 0; j < slice.length; j++) {\n const storageDocId = encodeModelFirestoreDocId(slice[j]!.modelId);\n const data = snaps[j]!.data() as ExistingCatalogDoc | undefined;\n if (data) map.set(storageDocId, data);\n }\n }\n return map;\n}\n\nexport type UpsertAiModelRecordOptions = {\n firestore: Firestore;\n context: CataloxContext;\n catalogId: string;\n record: AiModelRecord;\n};\n\n/**\n * Upserts one ai-models row via Firestore (safe doc ids for `provider/model` ids).\n */\nexport async function upsertAiModelRecord(options: UpsertAiModelRecordOptions): Promise<void> {\n const { record, catalogId, context, firestore } = options;\n const appId = context.appId ?? \"ai-tools\";\n\n const layout = await resolveNativeItemsLayout(firestore, catalogId);\n const col =\n layout === \"legacy\"\n ? legacyNativeItemsCollectionRef(firestore, catalogId)\n : flatNativeItemsCollectionRef(firestore, catalogId);\n\n const storageDocId = encodeModelFirestoreDocId(record.modelId);\n const existing = (await col.doc(storageDocId).get()).data() as ExistingCatalogDoc | undefined;\n const now = new Date().toISOString();\n const payload = buildCatalogItemUpsertPayload(record, existing, { catalogId, appId, now });\n\n await col.doc(storageDocId).set(payload, { merge: true });\n}\n\nconst BATCH_SIZE = 400;\nconst DEFAULT_CHUNK_RETRIES = 3;\nconst RETRY_BASE_MS = 500;\n\nexport type BatchUpsertProgress = {\n completed: number;\n total: number;\n phase: \"loading-existing\" | \"writing\";\n};\n\nexport type BatchUpsertAiModelRecordsOptions = {\n onProgress?: (progress: BatchUpsertProgress) => void;\n maxChunkRetries?: number;\n};\n\nexport type BatchUpsertResult = {\n upserted: number;\n failed: Array<{ modelId: string; error: string }>;\n};\n\nfunction sleep(ms: number): Promise<void> {\n return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\nasync function commitChunkWithRetry(\n firestore: Firestore,\n col: ReturnType<typeof flatNativeItemsCollectionRef>,\n chunk: AiModelRecord[],\n existingByStorageId: Map<string, ExistingCatalogDoc>,\n options: {\n catalogId: string;\n appId: string;\n now: string;\n maxChunkRetries: number;\n },\n): Promise<{ upserted: number; failed: Array<{ modelId: string; error: string }> }> {\n const { catalogId, appId, now, maxChunkRetries } = options;\n for (let attempt = 0; attempt < maxChunkRetries; attempt++) {\n try {\n const batch = firestore.batch();\n for (const record of chunk) {\n const storageDocId = encodeModelFirestoreDocId(record.modelId);\n const existing = existingByStorageId.get(storageDocId);\n const payload = buildCatalogItemUpsertPayload(record, existing, { catalogId, appId, now });\n batch.set(col.doc(storageDocId), payload, { merge: true });\n }\n await batch.commit();\n return { upserted: chunk.length, failed: [] };\n } catch {\n if (attempt < maxChunkRetries - 1) {\n await sleep(RETRY_BASE_MS * 2 ** attempt);\n }\n }\n }\n\n const failed: Array<{ modelId: string; error: string }> = [];\n let upserted = 0;\n\n for (const record of chunk) {\n try {\n const storageDocId = encodeModelFirestoreDocId(record.modelId);\n const existing = existingByStorageId.get(storageDocId);\n const payload = buildCatalogItemUpsertPayload(record, existing, { catalogId, appId, now });\n await col.doc(storageDocId).set(payload, { merge: true });\n upserted++;\n } catch (error) {\n failed.push({\n modelId: record.modelId,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n\n return { upserted, failed };\n}\n\nexport async function batchUpsertAiModelRecords(\n firestore: Firestore,\n catalogId: string,\n context: CataloxContext,\n records: AiModelRecord[],\n batchOptions: BatchUpsertAiModelRecordsOptions = {},\n): Promise<BatchUpsertResult> {\n const layout = await resolveNativeItemsLayout(firestore, catalogId);\n const col =\n layout === \"legacy\"\n ? legacyNativeItemsCollectionRef(firestore, catalogId)\n : flatNativeItemsCollectionRef(firestore, catalogId);\n\n const appId = context.appId ?? \"ai-tools\";\n const now = new Date().toISOString();\n const maxChunkRetries = batchOptions.maxChunkRetries ?? DEFAULT_CHUNK_RETRIES;\n const total = records.length;\n\n batchOptions.onProgress?.({ completed: 0, total, phase: \"loading-existing\" });\n const existingByStorageId = await loadExistingByStorageId(col, records);\n\n let completed = 0;\n let upserted = 0;\n const failed: Array<{ modelId: string; error: string }> = [];\n\n for (let i = 0; i < records.length; i += BATCH_SIZE) {\n const chunk = records.slice(i, i + BATCH_SIZE);\n const chunkResult = await commitChunkWithRetry(firestore, col, chunk, existingByStorageId, {\n catalogId,\n appId,\n now,\n maxChunkRetries,\n });\n upserted += chunkResult.upserted;\n failed.push(...chunkResult.failed);\n completed += chunk.length;\n batchOptions.onProgress?.({ completed, total, phase: \"writing\" });\n }\n\n return { upserted, failed };\n}\n"],"mappings":";AAIA,IAAM,KAAK;AACX,IAAM,eAAe;AAEd,SAAS,0BAA0B,SAAyB;AACjE,MAAI,CAAC,QAAQ,SAAS,GAAG,EAAG,QAAO;AACnC,SAAO,GAAG,EAAE,GAAG,YAAY,GAAG,OAAO,KAAK,SAAS,MAAM,EAAE,SAAS,WAAW,CAAC;AAClF;AAEO,SAAS,0BAA0B,OAAuB;AAC/D,QAAM,SAAS,GAAG,EAAE,GAAG,YAAY;AACnC,MAAI,MAAM,WAAW,MAAM,GAAG;AAC5B,WAAO,OAAO,KAAK,MAAM,MAAM,OAAO,MAAM,GAAG,WAAW,EAAE,SAAS,MAAM;AAAA,EAC7E;AACA,SAAO;AACT;;;AChBA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAKA,SAAS,qBAAwB,OAAa;AACnD,SAAO,KAAK,MAAM,KAAK,UAAU,KAAK,CAAC;AACzC;AAEA,SAAS,aAAa,QAAkE;AACtF,SAAO;AAAA,IACL,SAAS,OAAO;AAAA,IAChB,YAAY,OAAO;AAAA,IACnB,MAAM,OAAO;AAAA,IACb,eAAe,OAAO;AAAA,IACtB,eAAe,OAAO;AAAA,IACtB,QAAQ,OAAO;AAAA,IACf,uBAAuB,OAAO;AAAA,IAC9B,eAAe,OAAO;AAAA,IACtB,mBAAmB,OAAO;AAAA,IAC1B,mBAAmB,OAAO;AAAA,IAC1B,aAAa,OAAO;AAAA,IACpB,UAAU,OAAO;AAAA,EACnB;AACF;AAQO,SAAS,8BACd,QACA,UACA,SACA;AACA,QAAM,EAAE,WAAW,OAAO,IAAI,IAAI;AAClC,SAAO,qBAAqB;AAAA,IAC1B,QAAQ,OAAO;AAAA,IACf;AAAA,IACA,kBAAkB;AAAA,IAClB,SAAS,aAAa,MAAM;AAAA,IAC5B,MAAM,qBAAqB,MAAM;AAAA,IACjC,UAAU;AAAA,MACR,WAAW,OAAO,UAAU,UAAU,aAAa,GAAG;AAAA,MACtD,YAAY;AAAA,MACZ,WAAW,CAAC;AAAA,MACZ,UAAU,CAAC;AAAA,IACb;AAAA,IACA,UAAU,UAAU,WAAW,KAAK;AAAA,EACtC,CAAC;AACH;AAEA,IAAM,sBAAsB;AAE5B,eAAe,wBACb,KACA,SAC0C;AAC1C,QAAM,MAAM,oBAAI,IAAgC;AAChD,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK,qBAAqB;AAC5D,UAAM,QAAQ,QAAQ,MAAM,GAAG,IAAI,mBAAmB;AACtD,UAAM,QAAQ,MAAM,QAAQ;AAAA,MAC1B,MAAM,IAAI,CAAC,MAAM,IAAI,IAAI,0BAA0B,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC;AAAA,IACtE;AACA,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,YAAM,eAAe,0BAA0B,MAAM,CAAC,EAAG,OAAO;AAChE,YAAM,OAAO,MAAM,CAAC,EAAG,KAAK;AAC5B,UAAI,KAAM,KAAI,IAAI,cAAc,IAAI;AAAA,IACtC;AAAA,EACF;AACA,SAAO;AACT;AAYA,eAAsB,oBAAoB,SAAoD;AAC5F,QAAM,EAAE,QAAQ,WAAW,SAAS,UAAU,IAAI;AAClD,QAAM,QAAQ,QAAQ,SAAS;AAE/B,QAAM,SAAS,MAAM,yBAAyB,WAAW,SAAS;AAClE,QAAM,MACJ,WAAW,WACP,+BAA+B,WAAW,SAAS,IACnD,6BAA6B,WAAW,SAAS;AAEvD,QAAM,eAAe,0BAA0B,OAAO,OAAO;AAC7D,QAAM,YAAY,MAAM,IAAI,IAAI,YAAY,EAAE,IAAI,GAAG,KAAK;AAC1D,QAAM,OAAM,oBAAI,KAAK,GAAE,YAAY;AACnC,QAAM,UAAU,8BAA8B,QAAQ,UAAU,EAAE,WAAW,OAAO,IAAI,CAAC;AAEzF,QAAM,IAAI,IAAI,YAAY,EAAE,IAAI,SAAS,EAAE,OAAO,KAAK,CAAC;AAC1D;AAEA,IAAM,aAAa;AACnB,IAAM,wBAAwB;AAC9B,IAAM,gBAAgB;AAkBtB,SAAS,MAAM,IAA2B;AACxC,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AACzD;AAEA,eAAe,qBACb,WACA,KACA,OACA,qBACA,SAMkF;AAClF,QAAM,EAAE,WAAW,OAAO,KAAK,gBAAgB,IAAI;AACnD,WAAS,UAAU,GAAG,UAAU,iBAAiB,WAAW;AAC1D,QAAI;AACF,YAAM,QAAQ,UAAU,MAAM;AAC9B,iBAAW,UAAU,OAAO;AAC1B,cAAM,eAAe,0BAA0B,OAAO,OAAO;AAC7D,cAAM,WAAW,oBAAoB,IAAI,YAAY;AACrD,cAAM,UAAU,8BAA8B,QAAQ,UAAU,EAAE,WAAW,OAAO,IAAI,CAAC;AACzF,cAAM,IAAI,IAAI,IAAI,YAAY,GAAG,SAAS,EAAE,OAAO,KAAK,CAAC;AAAA,MAC3D;AACA,YAAM,MAAM,OAAO;AACnB,aAAO,EAAE,UAAU,MAAM,QAAQ,QAAQ,CAAC,EAAE;AAAA,IAC9C,QAAQ;AACN,UAAI,UAAU,kBAAkB,GAAG;AACjC,cAAM,MAAM,gBAAgB,KAAK,OAAO;AAAA,MAC1C;AAAA,IACF;AAAA,EACF;AAEA,QAAM,SAAoD,CAAC;AAC3D,MAAI,WAAW;AAEf,aAAW,UAAU,OAAO;AAC1B,QAAI;AACF,YAAM,eAAe,0BAA0B,OAAO,OAAO;AAC7D,YAAM,WAAW,oBAAoB,IAAI,YAAY;AACrD,YAAM,UAAU,8BAA8B,QAAQ,UAAU,EAAE,WAAW,OAAO,IAAI,CAAC;AACzF,YAAM,IAAI,IAAI,YAAY,EAAE,IAAI,SAAS,EAAE,OAAO,KAAK,CAAC;AACxD;AAAA,IACF,SAAS,OAAO;AACd,aAAO,KAAK;AAAA,QACV,SAAS,OAAO;AAAA,QAChB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF;AAEA,SAAO,EAAE,UAAU,OAAO;AAC5B;AAEA,eAAsB,0BACpB,WACA,WACA,SACA,SACA,eAAiD,CAAC,GACtB;AAC5B,QAAM,SAAS,MAAM,yBAAyB,WAAW,SAAS;AAClE,QAAM,MACJ,WAAW,WACP,+BAA+B,WAAW,SAAS,IACnD,6BAA6B,WAAW,SAAS;AAEvD,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,OAAM,oBAAI,KAAK,GAAE,YAAY;AACnC,QAAM,kBAAkB,aAAa,mBAAmB;AACxD,QAAM,QAAQ,QAAQ;AAEtB,eAAa,aAAa,EAAE,WAAW,GAAG,OAAO,OAAO,mBAAmB,CAAC;AAC5E,QAAM,sBAAsB,MAAM,wBAAwB,KAAK,OAAO;AAEtE,MAAI,YAAY;AAChB,MAAI,WAAW;AACf,QAAM,SAAoD,CAAC;AAE3D,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK,YAAY;AACnD,UAAM,QAAQ,QAAQ,MAAM,GAAG,IAAI,UAAU;AAC7C,UAAM,cAAc,MAAM,qBAAqB,WAAW,KAAK,OAAO,qBAAqB;AAAA,MACzF;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AACD,gBAAY,YAAY;AACxB,WAAO,KAAK,GAAG,YAAY,MAAM;AACjC,iBAAa,MAAM;AACnB,iBAAa,aAAa,EAAE,WAAW,OAAO,OAAO,UAAU,CAAC;AAAA,EAClE;AAEA,SAAO,EAAE,UAAU,OAAO;AAC5B;","names":[]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
//# sourceMappingURL=chunk-M5TMA73F.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
|