@milaboratories/pl-middle-layer 1.64.33 → 1.64.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/mutator/template/template_cache.cjs +23 -0
- package/dist/mutator/template/template_cache.cjs.map +1 -1
- package/dist/mutator/template/template_cache.d.ts.map +1 -1
- package/dist/mutator/template/template_cache.js +24 -1
- package/dist/mutator/template/template_cache.js.map +1 -1
- package/dist/pool/driver.cjs.map +1 -1
- package/dist/pool/driver.js.map +1 -1
- package/package.json +13 -13
- package/src/mutator/template/template_cache.ts +36 -0
- package/src/pool/driver.ts +2 -2
|
@@ -51,6 +51,9 @@ function hashLibV3(lib) {
|
|
|
51
51
|
function hashSoftwareV3(sw) {
|
|
52
52
|
return (0, node_crypto.createHash)("sha256").update(_milaboratories_pl_model_backend.PlTemplateSoftwareV1.type.name).update(_milaboratories_pl_model_backend.PlTemplateSoftwareV1.type.version).update(sw.name).update(sw.version).update(sw.sourceHash).digest("hex");
|
|
53
53
|
}
|
|
54
|
+
function hashWasmV3(wasm) {
|
|
55
|
+
return (0, node_crypto.createHash)("sha256").update(_milaboratories_pl_model_backend.PlWasmV1.type.name).update(_milaboratories_pl_model_backend.PlWasmV1.type.version).update(wasm.name).update(wasm.version).update(wasm.sourceHash).digest("hex");
|
|
56
|
+
}
|
|
54
57
|
function flattenV2Tree(data) {
|
|
55
58
|
const nodes = [];
|
|
56
59
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -183,6 +186,18 @@ function flattenV3Tree(data) {
|
|
|
183
186
|
}
|
|
184
187
|
return hash;
|
|
185
188
|
}
|
|
189
|
+
function processWasm(wasm) {
|
|
190
|
+
const hash = hashWasmV3(wasm);
|
|
191
|
+
if (!seen.has(hash)) {
|
|
192
|
+
seen.add(hash);
|
|
193
|
+
nodes.push({
|
|
194
|
+
hash,
|
|
195
|
+
create: (tx) => tx.createValue(_milaboratories_pl_model_backend.PlWasmV1.type, JSON.stringify(_milaboratories_pl_model_backend.PlWasmV1.fromV3Data(wasm, getSourceCode(wasm.name, sources, wasm.sourceHash)).data)),
|
|
196
|
+
childHashes: []
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
return hash;
|
|
200
|
+
}
|
|
186
201
|
function processTemplate(tpl) {
|
|
187
202
|
const childHashes = [];
|
|
188
203
|
const children = [];
|
|
@@ -218,6 +233,14 @@ function flattenV3Tree(data) {
|
|
|
218
233
|
hash: h
|
|
219
234
|
});
|
|
220
235
|
}
|
|
236
|
+
for (const [wasmId, wasm] of Object.entries(tpl.wasm ?? {})) {
|
|
237
|
+
const h = processWasm(wasm);
|
|
238
|
+
childHashes.push(h);
|
|
239
|
+
children.push({
|
|
240
|
+
fieldName: `${_milaboratories_pl_model_backend.PlTemplateV1.wasmPrefix}/${wasmId}`,
|
|
241
|
+
hash: h
|
|
242
|
+
});
|
|
243
|
+
}
|
|
221
244
|
const h = (0, node_crypto.createHash)("sha256").update(_milaboratories_pl_model_backend.PlTemplateV1.type.name).update(_milaboratories_pl_model_backend.PlTemplateV1.type.version).update(tpl.hashOverride ?? "no-override").update(tpl.name).update(tpl.version).update(tpl.sourceHash);
|
|
222
245
|
for (const child of children) h.update("child:" + child.fieldName + ":" + child.hash);
|
|
223
246
|
const hash = h.digest("hex");
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"template_cache.cjs","names":["PlTemplateLibV1","PlTemplateSoftwareV1","PlTemplateV1","PlTemplateOverrideV1","getDebugFlags"],"sources":["../../../src/mutator/template/template_cache.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport type {\n AnyResourceRef,\n PlClient,\n PlTransaction,\n SignedResourceId,\n ResourceRef,\n} from \"@milaboratories/pl-client\";\nimport {\n ensureSignedResourceIdNotNull,\n field,\n resourceType,\n toGlobalResourceId,\n} from \"@milaboratories/pl-client\";\nimport {\n parseTemplate,\n PlTemplateLibV1,\n PlTemplateOverrideV1,\n PlTemplateSoftwareV1,\n PlTemplateV1,\n} from \"@milaboratories/pl-model-backend\";\nimport type {\n CompiledTemplateV3,\n TemplateData,\n TemplateDataV3,\n TemplateLibData,\n TemplateLibDataV3,\n TemplateSoftwareData,\n TemplateSoftwareDataV3,\n} from \"@milaboratories/pl-model-backend\";\nimport { notEmpty } from \"@milaboratories/ts-helpers\";\nimport type { BlockPackSpecPrepared } from \"../../model\";\nimport type { TemplateSpecPrepared } from \"../../model/template_spec\";\nimport { getDebugFlags } from \"../../debug\";\n\nexport const TemplateCacheType = resourceType(\"TemplateCache\", \"1\");\n\nexport const TemplateCacheFieldName = \"__templateCache\";\nconst BATCH_SIZE = 50;\n/** @internal exported for testing */\nexport const GC_ACCESS_THRESHOLD = 30;\n/** @internal exported for testing */\nexport const GC_MAX_ENTRIES = 3000;\n/** @internal exported for testing */\nexport const ACCESS_COUNT_KEY = \"_accessCount\";\n/** @internal exported for testing */\nexport const ACCESS_KEY_PREFIX = \"access_\";\n\n// ─── Stats ───────────────────────────────────────────────────────────────────\n\nexport type TemplateCacheStat = {\n totalMs: number;\n flattenMs: number;\n cacheInitMs: number;\n materializeMs: number;\n totalNodes: number;\n cacheHits: number;\n cacheMisses: number;\n batchCount: number;\n happyPath: boolean;\n gcTriggered: boolean;\n retries: number;\n templateFormat: string;\n};\n\nfunction initialStat(): TemplateCacheStat {\n return {\n totalMs: 0,\n flattenMs: 0,\n cacheInitMs: 0,\n materializeMs: 0,\n totalNodes: 0,\n cacheHits: 0,\n cacheMisses: 0,\n batchCount: 0,\n happyPath: false,\n gcTriggered: false,\n retries: 0,\n templateFormat: \"\",\n };\n}\n\n// ─── Tree node abstraction ───────────────────────────────────────────────────\n\ninterface CacheableNode {\n /** SHA-256 content hash (includes all descendant content) */\n hash: string;\n /** Creates this node's resource in a transaction.\n * childRefs maps child hash → already-resolved ResourceRef or SignedResourceId */\n create: (tx: PlTransaction, childRefs: ReadonlyMap<string, AnyResourceRef>) => ResourceRef;\n /** Hashes of direct child nodes this node depends on */\n childHashes: string[];\n}\n\n// ─── Hash computation helpers ────────────────────────────────────────────────\n\nfunction getSourceCode(name: string, sources: Record<string, string>, sourceHash: string): string {\n return notEmpty(\n sources[sourceHash],\n `trying to get \"${name}\" source: sources map doesn't contain source hash ${sourceHash}`,\n );\n}\n\n/**\n * Bottom-up hash composition: each node hashes its OWN content + child hash STRINGS.\n * This means each unique node is hashed exactly once → O(n) instead of O(n * depth).\n */\n\n// V2 leaf hashes (libs, software — no children)\n\nfunction hashLibV2(lib: TemplateLibData): string {\n return createHash(\"sha256\")\n .update(PlTemplateLibV1.type.name)\n .update(PlTemplateLibV1.type.version)\n .update(lib.name)\n .update(lib.version)\n .update(lib.src)\n .digest(\"hex\");\n}\n\nfunction hashSoftwareV2(sw: TemplateSoftwareData): string {\n return createHash(\"sha256\")\n .update(PlTemplateSoftwareV1.type.name)\n .update(PlTemplateSoftwareV1.type.version)\n .update(sw.name)\n .update(sw.version)\n .update(sw.src)\n .digest(\"hex\");\n}\n\n// V3 leaf hashes — use sourceHash directly instead of resolving source content\n\nfunction hashLibV3(lib: TemplateLibDataV3): string {\n return createHash(\"sha256\")\n .update(PlTemplateLibV1.type.name)\n .update(PlTemplateLibV1.type.version)\n .update(lib.name)\n .update(lib.version)\n .update(lib.sourceHash)\n .digest(\"hex\");\n}\n\nfunction hashSoftwareV3(sw: TemplateSoftwareDataV3): string {\n return createHash(\"sha256\")\n .update(PlTemplateSoftwareV1.type.name)\n .update(PlTemplateSoftwareV1.type.version)\n .update(sw.name)\n .update(sw.version)\n .update(sw.sourceHash)\n .digest(\"hex\");\n}\n\n// ─── Tree flattening ─────────────────────────────────────────────────────────\n\nfunction flattenV2Tree(data: TemplateData): CacheableNode[] {\n const nodes: CacheableNode[] = [];\n const seen = new Set<string>();\n\n function processLib(lib: TemplateLibData): string {\n const hash = hashLibV2(lib);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) =>\n tx.createValue(\n PlTemplateLibV1.type,\n JSON.stringify(PlTemplateLibV1.fromV2Data(lib).data),\n ),\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processSoftware(sw: TemplateSoftwareData): string {\n const hash = hashSoftwareV2(sw);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) => {\n const swData = PlTemplateSoftwareV1.fromV2Data(sw);\n const ref = tx.createStruct(PlTemplateSoftwareV1.type, swData.data);\n tx.setKValue(ref, PlTemplateSoftwareV1.metaNameKey, JSON.stringify(swData.name));\n tx.lock(ref);\n return ref;\n },\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processTemplate(tpl: TemplateData): string {\n // Process children first (bottom-up) — their hashes are computed before ours\n const childHashes: string[] = [];\n const children: { fieldName: string; hash: string }[] = [];\n\n for (const [libId, lib] of Object.entries(tpl.libs ?? {})) {\n const h = processLib(lib);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.libPrefix}/${libId}`, hash: h });\n }\n for (const [swId, sw] of Object.entries(tpl.software ?? {})) {\n const h = processSoftware(sw);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.softPrefix}/${swId}`, hash: h });\n }\n for (const [swId, sw] of Object.entries(tpl.assets ?? {})) {\n const h = processSoftware(sw);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.softPrefix}/${swId}`, hash: h });\n }\n for (const [tplId, sub] of Object.entries(tpl.templates ?? {})) {\n const h = processTemplate(sub);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.tplPrefix}/${tplId}`, hash: h });\n }\n\n // Compose hash from own content + child hash strings (NOT child content)\n const h = createHash(\"sha256\")\n .update(PlTemplateV1.type.name)\n .update(PlTemplateV1.type.version)\n .update(tpl.hashOverride ?? \"no-override\")\n .update(tpl.name)\n .update(tpl.version)\n .update(tpl.src);\n for (const child of children) {\n h.update(\"child:\" + child.fieldName + \":\" + child.hash);\n }\n const hash = h.digest(\"hex\");\n\n if (seen.has(hash)) return hash;\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx, childRefs) => {\n const tplRef = tx.createStruct(\n PlTemplateV1.type,\n JSON.stringify(PlTemplateV1.fromV2Data(tpl).data),\n );\n for (const child of children) {\n const fld = field(tplRef, child.fieldName);\n tx.createField(fld, \"Input\");\n tx.setField(fld, notEmpty(childRefs.get(child.hash), `missing child ref ${child.hash}`));\n }\n tx.lock(tplRef);\n\n if (!tpl.hashOverride) return tplRef;\n\n const overrideRef = tx.createStruct(\n PlTemplateOverrideV1.type,\n JSON.stringify(PlTemplateOverrideV1.fromV2Data(tpl)),\n );\n const overrideFld = PlTemplateOverrideV1.tplField(overrideRef);\n tx.createField(overrideFld, \"Service\");\n tx.setField(overrideFld, tplRef);\n tx.lock(overrideRef);\n return overrideRef;\n },\n childHashes,\n });\n\n return hash;\n }\n\n processTemplate(data);\n return nodes;\n}\n\nfunction flattenV3Tree(data: CompiledTemplateV3): CacheableNode[] {\n const nodes: CacheableNode[] = [];\n const seen = new Set<string>();\n const sources = data.hashToSource;\n\n function processLib(lib: TemplateLibDataV3): string {\n const hash = hashLibV3(lib);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) =>\n tx.createValue(\n PlTemplateLibV1.type,\n JSON.stringify(\n PlTemplateLibV1.fromV3Data(lib, getSourceCode(lib.name, sources, lib.sourceHash))\n .data,\n ),\n ),\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processSoftware(sw: TemplateSoftwareDataV3): string {\n const hash = hashSoftwareV3(sw);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) => {\n const swData = PlTemplateSoftwareV1.fromV3Data(\n sw,\n getSourceCode(sw.name, sources, sw.sourceHash),\n );\n const ref = tx.createStruct(PlTemplateSoftwareV1.type, swData.data);\n tx.setKValue(ref, PlTemplateSoftwareV1.metaNameKey, JSON.stringify(swData.name));\n tx.lock(ref);\n return ref;\n },\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processTemplate(tpl: TemplateDataV3): string {\n // Process children first (bottom-up)\n const childHashes: string[] = [];\n const children: { fieldName: string; hash: string }[] = [];\n\n for (const [libId, lib] of Object.entries(tpl.libs ?? {})) {\n const h = processLib(lib);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.libPrefix}/${libId}`, hash: h });\n }\n for (const [swId, sw] of Object.entries(tpl.software ?? {})) {\n const h = processSoftware(sw);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.softPrefix}/${swId}`, hash: h });\n }\n for (const [swId, sw] of Object.entries(tpl.assets ?? {})) {\n const h = processSoftware(sw);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.softPrefix}/${swId}`, hash: h });\n }\n for (const [tplId, sub] of Object.entries(tpl.templates ?? {})) {\n const h = processTemplate(sub);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.tplPrefix}/${tplId}`, hash: h });\n }\n\n // Compose hash from own content + child hash strings (NOT child content).\n // Uses sourceHash directly — it already uniquely identifies the source.\n const h = createHash(\"sha256\")\n .update(PlTemplateV1.type.name)\n .update(PlTemplateV1.type.version)\n .update(tpl.hashOverride ?? \"no-override\")\n .update(tpl.name)\n .update(tpl.version)\n .update(tpl.sourceHash);\n for (const child of children) {\n h.update(\"child:\" + child.fieldName + \":\" + child.hash);\n }\n const hash = h.digest(\"hex\");\n\n if (seen.has(hash)) return hash;\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx, childRefs) => {\n const sourceCode = getSourceCode(tpl.name, sources, tpl.sourceHash);\n const tplRef = tx.createStruct(\n PlTemplateV1.type,\n JSON.stringify(PlTemplateV1.fromV3Data(tpl, sourceCode).data),\n );\n for (const child of children) {\n const fld = field(tplRef, child.fieldName);\n tx.createField(fld, \"Input\");\n tx.setField(fld, notEmpty(childRefs.get(child.hash), `missing child ref ${child.hash}`));\n }\n tx.lock(tplRef);\n\n if (!tpl.hashOverride) return tplRef;\n\n const overrideRef = tx.createStruct(\n PlTemplateOverrideV1.type,\n JSON.stringify(PlTemplateOverrideV1.fromV3Data(tpl)),\n );\n const overrideFld = PlTemplateOverrideV1.tplField(overrideRef);\n tx.createField(overrideFld, \"Service\");\n tx.setField(overrideFld, tplRef);\n tx.lock(overrideRef);\n return overrideRef;\n },\n childHashes,\n });\n\n return hash;\n }\n\n processTemplate(data.template);\n return nodes;\n}\n\n/** Flatten template tree into a topologically ordered list of cacheable nodes (leaves first). */\nexport function flattenTemplateTree(data: TemplateData | CompiledTemplateV3): CacheableNode[] {\n if (data.type === \"pl.tengo-template.v2\") {\n return flattenV2Tree(data);\n } else {\n return flattenV3Tree(data);\n }\n}\n\n// ─── Cache operations ────────────────────────────────────────────────────────\n\n/** In-memory cache for the TemplateCache SignedResourceId per PlClient instance. */\nconst cacheRidMap = new WeakMap<PlClient, SignedResourceId>();\n\n/** Clear the in-memory cacheRid entry (call on errors referencing the cache resource). */\nexport function invalidateTemplateCacheId(pl: PlClient): void {\n cacheRidMap.delete(pl);\n}\n\n/** Find or create the TemplateCache/1 resource on user root. */\nexport async function getOrCreateTemplateCache(pl: PlClient): Promise<SignedResourceId> {\n // Check in-memory cache first (0ms after first call)\n const cached = cacheRidMap.get(pl);\n if (cached !== undefined) return cached;\n\n // Try read-only check\n const existing = await pl.withReadTx(\"templateCache:check\", async (tx) => {\n const fd = await tx.getFieldIfExists(field(pl.clientRoot, TemplateCacheFieldName));\n return fd ? ensureSignedResourceIdNotNull(fd.value) : undefined;\n });\n if (existing) {\n cacheRidMap.set(pl, existing);\n return existing;\n }\n\n const result = await pl.withWriteTx(\"templateCache:init\", async (tx) => {\n // Double-check inside write tx (another instance may have created it)\n const fd = await tx.getFieldIfExists(field(pl.clientRoot, TemplateCacheFieldName));\n if (fd) return ensureSignedResourceIdNotNull(fd.value);\n\n const cache = tx.createStruct(TemplateCacheType);\n tx.createField(field(pl.clientRoot, TemplateCacheFieldName), \"Dynamic\", cache);\n tx.lock(cache);\n await tx.commit();\n return await cache.globalId;\n });\n cacheRidMap.set(pl, result);\n return result;\n}\n\n/** Remove the template cache from user root. */\nexport async function dropTemplateCache(pl: PlClient): Promise<void> {\n await pl.withWriteTx(\"templateCache:drop\", async (tx) => {\n const cacheField = field(pl.clientRoot, TemplateCacheFieldName);\n const fd = await tx.getFieldIfExists(cacheField);\n if (fd) {\n tx.removeField(cacheField);\n await tx.commit();\n }\n });\n invalidateTemplateCacheId(pl);\n}\n\n// ─── GC ──────────────────────────────────────────────────────────────────────\n\n/**\n * Run count-based garbage collection on the template cache.\n * Evicts least-recently-used entries when the cache exceeds maxEntries.\n * Always resets the access counter to 0.\n *\n * @internal exported for testing (maxEntries parameter allows low thresholds in tests)\n * @returns true if entries were evicted\n */\nexport async function runGc(\n pl: PlClient,\n cacheRid: SignedResourceId,\n maxEntries: number = GC_MAX_ENTRIES,\n): Promise<boolean> {\n return await pl.withWriteTx(\"templateCache:gc\", async (tx) => {\n const kvs = await tx.listKeyValuesString(cacheRid);\n const entries: { hash: string; timestamp: number }[] = [];\n for (const { key, value } of kvs) {\n if (!key.startsWith(ACCESS_KEY_PREFIX)) continue;\n entries.push({\n hash: key.slice(ACCESS_KEY_PREFIX.length),\n timestamp: parseInt(value, 10),\n });\n }\n\n // Always reset counter\n tx.setKValue(cacheRid, ACCESS_COUNT_KEY, \"0\");\n\n if (entries.length <= maxEntries) {\n await tx.commit();\n return false;\n }\n\n // Sort oldest first, evict until under limit\n entries.sort((a, b) => a.timestamp - b.timestamp);\n const toEvict = entries.length - maxEntries;\n for (let i = 0; i < toEvict; i++) {\n tx.removeField(field(cacheRid, entries[i].hash));\n tx.deleteKValue(cacheRid, ACCESS_KEY_PREFIX + entries[i].hash);\n }\n\n await tx.commit();\n return true;\n });\n}\n\n// ─── Batched materialization ─────────────────────────────────────────────────\n\n/** Create a batch of cache nodes in the current transaction. */\nfunction createBatchNodes(\n tx: PlTransaction,\n cacheRid: SignedResourceId,\n batch: CacheableNode[],\n resolvedIds: ReadonlyMap<string, SignedResourceId>,\n newRefs: Map<string, ResourceRef>,\n now: string,\n): void {\n for (const node of batch) {\n const childRefs = new Map<string, AnyResourceRef>();\n for (const ch of node.childHashes) {\n const resolved = resolvedIds.get(ch) ?? newRefs.get(ch);\n if (resolved === undefined) {\n throw new Error(`BUG: child ${ch} not resolved`);\n }\n childRefs.set(ch, resolved);\n }\n const ref = node.create(tx, childRefs);\n newRefs.set(node.hash, ref);\n tx.createField(field(cacheRid, node.hash), \"Dynamic\", ref);\n tx.setKValue(cacheRid, ACCESS_KEY_PREFIX + node.hash, now);\n }\n}\n\n/**\n * Materialize a template tree via the cache using \"probe all + batched creation\".\n *\n * Phase 1 (single write tx):\n * - Check existence of ALL hashes in one roundtrip\n * - Happy path: if root cached, update access tracking and return\n * - Otherwise: fetch ResourceIds for all cache hits, create first batch of missing nodes\n *\n * Phase 2..N (one write tx per batch):\n * - Create remaining missing nodes in BATCH_SIZE chunks\n *\n * @returns root SignedResourceId and current access count (for GC decision)\n */\nasync function materialize(\n pl: PlClient,\n cacheRid: SignedResourceId,\n rootHash: string,\n nodes: CacheableNode[],\n stat: TemplateCacheStat,\n): Promise<{ rootId: SignedResourceId; accessCount: number }> {\n const allHashes = nodes.map((n) => n.hash);\n const resolvedIds = new Map<string, SignedResourceId>();\n\n // Phase 1: probe all + first batch\n const phase1 = await pl.withWriteTx(\"templateCache:materialize\", async (tx) => {\n // 1 roundtrip: check all hashes + read access count\n const [exists, countStr] = await Promise.all([\n Promise.all(allHashes.map((h) => tx.fieldExists(field(cacheRid, h)))),\n tx.getKValueStringIfExists(cacheRid, ACCESS_COUNT_KEY),\n ]);\n\n const prevCount = countStr ? parseInt(countStr, 10) : 0;\n const newCount = prevCount + 1;\n const now = Date.now().toString();\n const rootIdx = allHashes.length - 1;\n\n // Happy path: root already cached\n if (exists[rootIdx]) {\n const rootFd = await tx.getField(field(cacheRid, rootHash));\n const rootRid = ensureSignedResourceIdNotNull(rootFd.value);\n tx.setKValue(cacheRid, ACCESS_KEY_PREFIX + rootHash, now);\n tx.setKValue(cacheRid, ACCESS_COUNT_KEY, newCount.toString());\n await tx.commit();\n stat.happyPath = true;\n stat.cacheHits = stat.totalNodes;\n stat.batchCount = 1;\n return { done: true as const, rootId: rootRid, accessCount: newCount };\n }\n\n // Fetch ResourceIds for all cache hits (1 roundtrip)\n const hitIndices: number[] = [];\n for (let i = 0; i < allHashes.length; i++) {\n if (exists[i]) hitIndices.push(i);\n }\n\n if (hitIndices.length > 0) {\n const hitFields = await Promise.all(\n hitIndices.map((i) => tx.getField(field(cacheRid, allHashes[i]))),\n );\n for (let j = 0; j < hitIndices.length; j++) {\n resolvedIds.set(\n allHashes[hitIndices[j]],\n ensureSignedResourceIdNotNull(hitFields[j].value),\n );\n }\n }\n stat.cacheHits = hitIndices.length;\n\n // Missing nodes (topo order preserved from flatten)\n const missing = nodes.filter((n) => !resolvedIds.has(n.hash));\n stat.cacheMisses = missing.length;\n\n // Create first batch of missing nodes\n const firstBatch = missing.slice(0, BATCH_SIZE);\n const newRefs = new Map<string, ResourceRef>();\n createBatchNodes(tx, cacheRid, firstBatch, resolvedIds, newRefs, now);\n\n // Update access tracking for cache hits\n for (const i of hitIndices) {\n tx.setKValue(cacheRid, ACCESS_KEY_PREFIX + allHashes[i], now);\n }\n tx.setKValue(cacheRid, ACCESS_COUNT_KEY, newCount.toString());\n\n await tx.commit();\n\n // Resolve new refs to global IDs (after commit)\n for (const [hash, ref] of newRefs) {\n resolvedIds.set(hash, await toGlobalResourceId(ref));\n }\n\n return {\n done: false as const,\n remaining: missing.slice(BATCH_SIZE),\n accessCount: newCount,\n };\n });\n\n if (phase1.done) {\n return { rootId: phase1.rootId, accessCount: phase1.accessCount };\n }\n\n stat.batchCount = 1;\n\n // Phase 2+: remaining batches\n const { remaining } = phase1;\n for (let i = 0; i < remaining.length; i += BATCH_SIZE) {\n const batch = remaining.slice(i, i + BATCH_SIZE);\n stat.batchCount++;\n\n await pl.withWriteTx(\"templateCache:create\", async (tx) => {\n const newRefs = new Map<string, ResourceRef>();\n const now = Date.now().toString();\n createBatchNodes(tx, cacheRid, batch, resolvedIds, newRefs, now);\n await tx.commit();\n\n for (const [hash, ref] of newRefs) {\n resolvedIds.set(hash, await toGlobalResourceId(ref));\n }\n });\n }\n\n const rootId = resolvedIds.get(rootHash);\n if (!rootId) throw new Error(\"BUG: root hash not resolved after all batches\");\n return { rootId, accessCount: phase1.accessCount };\n}\n\n/**\n * Materialize a template tree via the cache.\n * Manages its own transactions internally — do NOT call inside an existing tx.\n *\n * @returns concrete SignedResourceId of the root template\n */\nexport async function loadTemplateCached(\n pl: PlClient,\n spec: TemplateSpecPrepared,\n options?: { cacheResourceId?: SignedResourceId },\n): Promise<SignedResourceId> {\n const stat = initialStat();\n const t0 = performance.now();\n\n try {\n // Parse to data if needed\n let tplData: TemplateData | CompiledTemplateV3;\n switch (spec.type) {\n case \"explicit\":\n tplData = parseTemplate(spec.content);\n break;\n case \"prepared\":\n tplData = spec.data;\n break;\n case \"cached\":\n return spec.resourceId;\n case \"from-registry\":\n throw new Error(\n \"loadTemplateCached does not support from-registry specs; use loadTemplate instead\",\n );\n default: {\n const _: never = spec;\n throw new Error(`unexpected spec type: ${(_ as any).type}`);\n }\n }\n\n stat.templateFormat = tplData.type;\n\n // Flatten to ordered nodes\n const tFlatten = performance.now();\n const nodes = flattenTemplateTree(tplData);\n stat.flattenMs = performance.now() - tFlatten;\n if (nodes.length === 0) throw new Error(\"template tree produced no nodes\");\n\n stat.totalNodes = nodes.length;\n const rootHash = nodes[nodes.length - 1].hash;\n\n // Resolve or create cache resource\n const tCacheInit = performance.now();\n const cacheRid = options?.cacheResourceId ?? (await getOrCreateTemplateCache(pl));\n stat.cacheInitMs = performance.now() - tCacheInit;\n\n // Retry loop: if a write tx fails (e.g. concurrent GC invalidated a cached resource),\n // restart materialization from scratch.\n const MAX_RETRIES = 3;\n for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {\n try {\n const tMat = performance.now();\n const result = await materialize(pl, cacheRid, rootHash, nodes, stat);\n stat.materializeMs = performance.now() - tMat;\n stat.retries = attempt;\n\n // GC in separate tx if access count exceeded threshold\n if (result.accessCount >= GC_ACCESS_THRESHOLD) {\n await runGc(pl, cacheRid);\n stat.gcTriggered = true;\n }\n\n return result.rootId;\n } catch (e) {\n if (attempt === MAX_RETRIES - 1) throw e;\n // Retry from scratch — previous batch results may reference GC'd resources\n stat.cacheHits = 0;\n stat.cacheMisses = 0;\n }\n }\n\n throw new Error(\"BUG: unreachable\");\n } finally {\n stat.totalMs = performance.now() - t0;\n if (getDebugFlags().logTemplateCacheStat) {\n console.log(`[templateCache] ${JSON.stringify(stat)}`);\n }\n }\n}\n\n// ─── Caller helper ───────────────────────────────────────────────────────────\n\n/**\n * Pre-materialize a block pack's template via cache.\n * Returns a copy of the spec with the template replaced by a cached reference.\n * If the template is already cached, returns the spec unchanged.\n */\nexport async function cacheBlockPackTemplate(\n pl: PlClient,\n spec: BlockPackSpecPrepared,\n options?: { cacheResourceId?: SignedResourceId },\n): Promise<BlockPackSpecPrepared> {\n if (spec.template.type === \"cached\") return spec;\n\n const resourceId = await loadTemplateCached(pl, spec.template, options);\n return {\n ...spec,\n template: { type: \"cached\", resourceId },\n };\n}\n"],"mappings":";;;;;;;AAmCA,MAAa,qBAAA,GAAA,0BAAA,cAAiC,iBAAiB,IAAI;AAEnE,MAAa,yBAAyB;AACtC,MAAM,aAAa;;AAEnB,MAAa,sBAAsB;;AAEnC,MAAa,iBAAiB;;AAE9B,MAAa,mBAAmB;;AAEhC,MAAa,oBAAoB;AAmBjC,SAAS,cAAiC;AACxC,QAAO;EACL,SAAS;EACT,WAAW;EACX,aAAa;EACb,eAAe;EACf,YAAY;EACZ,WAAW;EACX,aAAa;EACb,YAAY;EACZ,WAAW;EACX,aAAa;EACb,SAAS;EACT,gBAAgB;EACjB;;AAiBH,SAAS,cAAc,MAAc,SAAiC,YAA4B;AAChG,SAAA,GAAA,2BAAA,UACE,QAAQ,aACR,kBAAkB,KAAK,oDAAoD,aAC5E;;;;;;AAUH,SAAS,UAAU,KAA8B;AAC/C,SAAA,GAAA,YAAA,YAAkB,SAAS,CACxB,OAAOA,iCAAAA,gBAAgB,KAAK,KAAK,CACjC,OAAOA,iCAAAA,gBAAgB,KAAK,QAAQ,CACpC,OAAO,IAAI,KAAK,CAChB,OAAO,IAAI,QAAQ,CACnB,OAAO,IAAI,IAAI,CACf,OAAO,MAAM;;AAGlB,SAAS,eAAe,IAAkC;AACxD,SAAA,GAAA,YAAA,YAAkB,SAAS,CACxB,OAAOC,iCAAAA,qBAAqB,KAAK,KAAK,CACtC,OAAOA,iCAAAA,qBAAqB,KAAK,QAAQ,CACzC,OAAO,GAAG,KAAK,CACf,OAAO,GAAG,QAAQ,CAClB,OAAO,GAAG,IAAI,CACd,OAAO,MAAM;;AAKlB,SAAS,UAAU,KAAgC;AACjD,SAAA,GAAA,YAAA,YAAkB,SAAS,CACxB,OAAOD,iCAAAA,gBAAgB,KAAK,KAAK,CACjC,OAAOA,iCAAAA,gBAAgB,KAAK,QAAQ,CACpC,OAAO,IAAI,KAAK,CAChB,OAAO,IAAI,QAAQ,CACnB,OAAO,IAAI,WAAW,CACtB,OAAO,MAAM;;AAGlB,SAAS,eAAe,IAAoC;AAC1D,SAAA,GAAA,YAAA,YAAkB,SAAS,CACxB,OAAOC,iCAAAA,qBAAqB,KAAK,KAAK,CACtC,OAAOA,iCAAAA,qBAAqB,KAAK,QAAQ,CACzC,OAAO,GAAG,KAAK,CACf,OAAO,GAAG,QAAQ,CAClB,OAAO,GAAG,WAAW,CACrB,OAAO,MAAM;;AAKlB,SAAS,cAAc,MAAqC;CAC1D,MAAM,QAAyB,EAAE;CACjC,MAAM,uBAAO,IAAI,KAAa;CAE9B,SAAS,WAAW,KAA8B;EAChD,MAAM,OAAO,UAAU,IAAI;AAC3B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OACP,GAAG,YACDD,iCAAAA,gBAAgB,MAChB,KAAK,UAAUA,iCAAAA,gBAAgB,WAAW,IAAI,CAAC,KAAK,CACrD;IACH,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,gBAAgB,IAAkC;EACzD,MAAM,OAAO,eAAe,GAAG;AAC/B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OAAO;KACd,MAAM,SAASC,iCAAAA,qBAAqB,WAAW,GAAG;KAClD,MAAM,MAAM,GAAG,aAAaA,iCAAAA,qBAAqB,MAAM,OAAO,KAAK;AACnE,QAAG,UAAU,KAAKA,iCAAAA,qBAAqB,aAAa,KAAK,UAAU,OAAO,KAAK,CAAC;AAChF,QAAG,KAAK,IAAI;AACZ,YAAO;;IAET,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,gBAAgB,KAA2B;EAElD,MAAM,cAAwB,EAAE;EAChC,MAAM,WAAkD,EAAE;AAE1D,OAAK,MAAM,CAAC,OAAO,QAAQ,OAAO,QAAQ,IAAI,QAAQ,EAAE,CAAC,EAAE;GACzD,MAAM,IAAI,WAAW,IAAI;AACzB,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAGC,iCAAAA,aAAa,UAAU,GAAG;IAAS,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,MAAM,OAAO,OAAO,QAAQ,IAAI,YAAY,EAAE,CAAC,EAAE;GAC3D,MAAM,IAAI,gBAAgB,GAAG;AAC7B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAGA,iCAAAA,aAAa,WAAW,GAAG;IAAQ,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,MAAM,OAAO,OAAO,QAAQ,IAAI,UAAU,EAAE,CAAC,EAAE;GACzD,MAAM,IAAI,gBAAgB,GAAG;AAC7B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAGA,iCAAAA,aAAa,WAAW,GAAG;IAAQ,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,OAAO,QAAQ,OAAO,QAAQ,IAAI,aAAa,EAAE,CAAC,EAAE;GAC9D,MAAM,IAAI,gBAAgB,IAAI;AAC9B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAGA,iCAAAA,aAAa,UAAU,GAAG;IAAS,MAAM;IAAG,CAAC;;EAI7E,MAAM,KAAA,GAAA,YAAA,YAAe,SAAS,CAC3B,OAAOA,iCAAAA,aAAa,KAAK,KAAK,CAC9B,OAAOA,iCAAAA,aAAa,KAAK,QAAQ,CACjC,OAAO,IAAI,gBAAgB,cAAc,CACzC,OAAO,IAAI,KAAK,CAChB,OAAO,IAAI,QAAQ,CACnB,OAAO,IAAI,IAAI;AAClB,OAAK,MAAM,SAAS,SAClB,GAAE,OAAO,WAAW,MAAM,YAAY,MAAM,MAAM,KAAK;EAEzD,MAAM,OAAO,EAAE,OAAO,MAAM;AAE5B,MAAI,KAAK,IAAI,KAAK,CAAE,QAAO;AAC3B,OAAK,IAAI,KAAK;AACd,QAAM,KAAK;GACT;GACA,SAAS,IAAI,cAAc;IACzB,MAAM,SAAS,GAAG,aAChBA,iCAAAA,aAAa,MACb,KAAK,UAAUA,iCAAAA,aAAa,WAAW,IAAI,CAAC,KAAK,CAClD;AACD,SAAK,MAAM,SAAS,UAAU;KAC5B,MAAM,OAAA,GAAA,0BAAA,OAAY,QAAQ,MAAM,UAAU;AAC1C,QAAG,YAAY,KAAK,QAAQ;AAC5B,QAAG,SAAS,MAAA,GAAA,2BAAA,UAAc,UAAU,IAAI,MAAM,KAAK,EAAE,qBAAqB,MAAM,OAAO,CAAC;;AAE1F,OAAG,KAAK,OAAO;AAEf,QAAI,CAAC,IAAI,aAAc,QAAO;IAE9B,MAAM,cAAc,GAAG,aACrBC,iCAAAA,qBAAqB,MACrB,KAAK,UAAUA,iCAAAA,qBAAqB,WAAW,IAAI,CAAC,CACrD;IACD,MAAM,cAAcA,iCAAAA,qBAAqB,SAAS,YAAY;AAC9D,OAAG,YAAY,aAAa,UAAU;AACtC,OAAG,SAAS,aAAa,OAAO;AAChC,OAAG,KAAK,YAAY;AACpB,WAAO;;GAET;GACD,CAAC;AAEF,SAAO;;AAGT,iBAAgB,KAAK;AACrB,QAAO;;AAGT,SAAS,cAAc,MAA2C;CAChE,MAAM,QAAyB,EAAE;CACjC,MAAM,uBAAO,IAAI,KAAa;CAC9B,MAAM,UAAU,KAAK;CAErB,SAAS,WAAW,KAAgC;EAClD,MAAM,OAAO,UAAU,IAAI;AAC3B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OACP,GAAG,YACDH,iCAAAA,gBAAgB,MAChB,KAAK,UACHA,iCAAAA,gBAAgB,WAAW,KAAK,cAAc,IAAI,MAAM,SAAS,IAAI,WAAW,CAAC,CAC9E,KACJ,CACF;IACH,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,gBAAgB,IAAoC;EAC3D,MAAM,OAAO,eAAe,GAAG;AAC/B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OAAO;KACd,MAAM,SAASC,iCAAAA,qBAAqB,WAClC,IACA,cAAc,GAAG,MAAM,SAAS,GAAG,WAAW,CAC/C;KACD,MAAM,MAAM,GAAG,aAAaA,iCAAAA,qBAAqB,MAAM,OAAO,KAAK;AACnE,QAAG,UAAU,KAAKA,iCAAAA,qBAAqB,aAAa,KAAK,UAAU,OAAO,KAAK,CAAC;AAChF,QAAG,KAAK,IAAI;AACZ,YAAO;;IAET,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,gBAAgB,KAA6B;EAEpD,MAAM,cAAwB,EAAE;EAChC,MAAM,WAAkD,EAAE;AAE1D,OAAK,MAAM,CAAC,OAAO,QAAQ,OAAO,QAAQ,IAAI,QAAQ,EAAE,CAAC,EAAE;GACzD,MAAM,IAAI,WAAW,IAAI;AACzB,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAGC,iCAAAA,aAAa,UAAU,GAAG;IAAS,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,MAAM,OAAO,OAAO,QAAQ,IAAI,YAAY,EAAE,CAAC,EAAE;GAC3D,MAAM,IAAI,gBAAgB,GAAG;AAC7B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAGA,iCAAAA,aAAa,WAAW,GAAG;IAAQ,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,MAAM,OAAO,OAAO,QAAQ,IAAI,UAAU,EAAE,CAAC,EAAE;GACzD,MAAM,IAAI,gBAAgB,GAAG;AAC7B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAGA,iCAAAA,aAAa,WAAW,GAAG;IAAQ,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,OAAO,QAAQ,OAAO,QAAQ,IAAI,aAAa,EAAE,CAAC,EAAE;GAC9D,MAAM,IAAI,gBAAgB,IAAI;AAC9B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAGA,iCAAAA,aAAa,UAAU,GAAG;IAAS,MAAM;IAAG,CAAC;;EAK7E,MAAM,KAAA,GAAA,YAAA,YAAe,SAAS,CAC3B,OAAOA,iCAAAA,aAAa,KAAK,KAAK,CAC9B,OAAOA,iCAAAA,aAAa,KAAK,QAAQ,CACjC,OAAO,IAAI,gBAAgB,cAAc,CACzC,OAAO,IAAI,KAAK,CAChB,OAAO,IAAI,QAAQ,CACnB,OAAO,IAAI,WAAW;AACzB,OAAK,MAAM,SAAS,SAClB,GAAE,OAAO,WAAW,MAAM,YAAY,MAAM,MAAM,KAAK;EAEzD,MAAM,OAAO,EAAE,OAAO,MAAM;AAE5B,MAAI,KAAK,IAAI,KAAK,CAAE,QAAO;AAC3B,OAAK,IAAI,KAAK;AACd,QAAM,KAAK;GACT;GACA,SAAS,IAAI,cAAc;IACzB,MAAM,aAAa,cAAc,IAAI,MAAM,SAAS,IAAI,WAAW;IACnE,MAAM,SAAS,GAAG,aAChBA,iCAAAA,aAAa,MACb,KAAK,UAAUA,iCAAAA,aAAa,WAAW,KAAK,WAAW,CAAC,KAAK,CAC9D;AACD,SAAK,MAAM,SAAS,UAAU;KAC5B,MAAM,OAAA,GAAA,0BAAA,OAAY,QAAQ,MAAM,UAAU;AAC1C,QAAG,YAAY,KAAK,QAAQ;AAC5B,QAAG,SAAS,MAAA,GAAA,2BAAA,UAAc,UAAU,IAAI,MAAM,KAAK,EAAE,qBAAqB,MAAM,OAAO,CAAC;;AAE1F,OAAG,KAAK,OAAO;AAEf,QAAI,CAAC,IAAI,aAAc,QAAO;IAE9B,MAAM,cAAc,GAAG,aACrBC,iCAAAA,qBAAqB,MACrB,KAAK,UAAUA,iCAAAA,qBAAqB,WAAW,IAAI,CAAC,CACrD;IACD,MAAM,cAAcA,iCAAAA,qBAAqB,SAAS,YAAY;AAC9D,OAAG,YAAY,aAAa,UAAU;AACtC,OAAG,SAAS,aAAa,OAAO;AAChC,OAAG,KAAK,YAAY;AACpB,WAAO;;GAET;GACD,CAAC;AAEF,SAAO;;AAGT,iBAAgB,KAAK,SAAS;AAC9B,QAAO;;;AAIT,SAAgB,oBAAoB,MAA0D;AAC5F,KAAI,KAAK,SAAS,uBAChB,QAAO,cAAc,KAAK;KAE1B,QAAO,cAAc,KAAK;;;AAO9B,MAAM,8BAAc,IAAI,SAAqC;;AAG7D,SAAgB,0BAA0B,IAAoB;AAC5D,aAAY,OAAO,GAAG;;;AAIxB,eAAsB,yBAAyB,IAAyC;CAEtF,MAAM,SAAS,YAAY,IAAI,GAAG;AAClC,KAAI,WAAW,KAAA,EAAW,QAAO;CAGjC,MAAM,WAAW,MAAM,GAAG,WAAW,uBAAuB,OAAO,OAAO;EACxE,MAAM,KAAK,MAAM,GAAG,kBAAA,GAAA,0BAAA,OAAuB,GAAG,YAAY,uBAAuB,CAAC;AAClF,SAAO,MAAA,GAAA,0BAAA,+BAAmC,GAAG,MAAM,GAAG,KAAA;GACtD;AACF,KAAI,UAAU;AACZ,cAAY,IAAI,IAAI,SAAS;AAC7B,SAAO;;CAGT,MAAM,SAAS,MAAM,GAAG,YAAY,sBAAsB,OAAO,OAAO;EAEtE,MAAM,KAAK,MAAM,GAAG,kBAAA,GAAA,0BAAA,OAAuB,GAAG,YAAY,uBAAuB,CAAC;AAClF,MAAI,GAAI,SAAA,GAAA,0BAAA,+BAAqC,GAAG,MAAM;EAEtD,MAAM,QAAQ,GAAG,aAAa,kBAAkB;AAChD,KAAG,aAAA,GAAA,0BAAA,OAAkB,GAAG,YAAY,uBAAuB,EAAE,WAAW,MAAM;AAC9E,KAAG,KAAK,MAAM;AACd,QAAM,GAAG,QAAQ;AACjB,SAAO,MAAM,MAAM;GACnB;AACF,aAAY,IAAI,IAAI,OAAO;AAC3B,QAAO;;;AAIT,eAAsB,kBAAkB,IAA6B;AACnE,OAAM,GAAG,YAAY,sBAAsB,OAAO,OAAO;EACvD,MAAM,cAAA,GAAA,0BAAA,OAAmB,GAAG,YAAY,uBAAuB;AAE/D,MADW,MAAM,GAAG,iBAAiB,WAAW,EACxC;AACN,MAAG,YAAY,WAAW;AAC1B,SAAM,GAAG,QAAQ;;GAEnB;AACF,2BAA0B,GAAG;;;;;;;;;;AAa/B,eAAsB,MACpB,IACA,UACA,aAAqB,gBACH;AAClB,QAAO,MAAM,GAAG,YAAY,oBAAoB,OAAO,OAAO;EAC5D,MAAM,MAAM,MAAM,GAAG,oBAAoB,SAAS;EAClD,MAAM,UAAiD,EAAE;AACzD,OAAK,MAAM,EAAE,KAAK,WAAW,KAAK;AAChC,OAAI,CAAC,IAAI,WAAA,UAA6B,CAAE;AACxC,WAAQ,KAAK;IACX,MAAM,IAAI,MAAM,EAAyB;IACzC,WAAW,SAAS,OAAO,GAAG;IAC/B,CAAC;;AAIJ,KAAG,UAAU,UAAU,kBAAkB,IAAI;AAE7C,MAAI,QAAQ,UAAU,YAAY;AAChC,SAAM,GAAG,QAAQ;AACjB,UAAO;;AAIT,UAAQ,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,UAAU;EACjD,MAAM,UAAU,QAAQ,SAAS;AACjC,OAAK,IAAI,IAAI,GAAG,IAAI,SAAS,KAAK;AAChC,MAAG,aAAA,GAAA,0BAAA,OAAkB,UAAU,QAAQ,GAAG,KAAK,CAAC;AAChD,MAAG,aAAa,UAAU,oBAAoB,QAAQ,GAAG,KAAK;;AAGhE,QAAM,GAAG,QAAQ;AACjB,SAAO;GACP;;;AAMJ,SAAS,iBACP,IACA,UACA,OACA,aACA,SACA,KACM;AACN,MAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,4BAAY,IAAI,KAA6B;AACnD,OAAK,MAAM,MAAM,KAAK,aAAa;GACjC,MAAM,WAAW,YAAY,IAAI,GAAG,IAAI,QAAQ,IAAI,GAAG;AACvD,OAAI,aAAa,KAAA,EACf,OAAM,IAAI,MAAM,cAAc,GAAG,eAAe;AAElD,aAAU,IAAI,IAAI,SAAS;;EAE7B,MAAM,MAAM,KAAK,OAAO,IAAI,UAAU;AACtC,UAAQ,IAAI,KAAK,MAAM,IAAI;AAC3B,KAAG,aAAA,GAAA,0BAAA,OAAkB,UAAU,KAAK,KAAK,EAAE,WAAW,IAAI;AAC1D,KAAG,UAAU,UAAU,oBAAoB,KAAK,MAAM,IAAI;;;;;;;;;;;;;;;;AAiB9D,eAAe,YACb,IACA,UACA,UACA,OACA,MAC4D;CAC5D,MAAM,YAAY,MAAM,KAAK,MAAM,EAAE,KAAK;CAC1C,MAAM,8BAAc,IAAI,KAA+B;CAGvD,MAAM,SAAS,MAAM,GAAG,YAAY,6BAA6B,OAAO,OAAO;EAE7E,MAAM,CAAC,QAAQ,YAAY,MAAM,QAAQ,IAAI,CAC3C,QAAQ,IAAI,UAAU,KAAK,MAAM,GAAG,aAAA,GAAA,0BAAA,OAAkB,UAAU,EAAE,CAAC,CAAC,CAAC,EACrE,GAAG,wBAAwB,UAAU,iBAAiB,CACvD,CAAC;EAGF,MAAM,YADY,WAAW,SAAS,UAAU,GAAG,GAAG,KACzB;EAC7B,MAAM,MAAM,KAAK,KAAK,CAAC,UAAU;AAIjC,MAAI,OAHY,UAAU,SAAS,IAGd;GAEnB,MAAM,WAAA,GAAA,0BAAA,gCADS,MAAM,GAAG,UAAA,GAAA,0BAAA,OAAe,UAAU,SAAS,CAAC,EACN,MAAM;AAC3D,MAAG,UAAU,UAAU,oBAAoB,UAAU,IAAI;AACzD,MAAG,UAAU,UAAU,kBAAkB,SAAS,UAAU,CAAC;AAC7D,SAAM,GAAG,QAAQ;AACjB,QAAK,YAAY;AACjB,QAAK,YAAY,KAAK;AACtB,QAAK,aAAa;AAClB,UAAO;IAAE,MAAM;IAAe,QAAQ;IAAS,aAAa;IAAU;;EAIxE,MAAM,aAAuB,EAAE;AAC/B,OAAK,IAAI,IAAI,GAAG,IAAI,UAAU,QAAQ,IACpC,KAAI,OAAO,GAAI,YAAW,KAAK,EAAE;AAGnC,MAAI,WAAW,SAAS,GAAG;GACzB,MAAM,YAAY,MAAM,QAAQ,IAC9B,WAAW,KAAK,MAAM,GAAG,UAAA,GAAA,0BAAA,OAAe,UAAU,UAAU,GAAG,CAAC,CAAC,CAClE;AACD,QAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,IACrC,aAAY,IACV,UAAU,WAAW,MAAA,GAAA,0BAAA,+BACS,UAAU,GAAG,MAAM,CAClD;;AAGL,OAAK,YAAY,WAAW;EAG5B,MAAM,UAAU,MAAM,QAAQ,MAAM,CAAC,YAAY,IAAI,EAAE,KAAK,CAAC;AAC7D,OAAK,cAAc,QAAQ;EAG3B,MAAM,aAAa,QAAQ,MAAM,GAAG,WAAW;EAC/C,MAAM,0BAAU,IAAI,KAA0B;AAC9C,mBAAiB,IAAI,UAAU,YAAY,aAAa,SAAS,IAAI;AAGrE,OAAK,MAAM,KAAK,WACd,IAAG,UAAU,UAAU,oBAAoB,UAAU,IAAI,IAAI;AAE/D,KAAG,UAAU,UAAU,kBAAkB,SAAS,UAAU,CAAC;AAE7D,QAAM,GAAG,QAAQ;AAGjB,OAAK,MAAM,CAAC,MAAM,QAAQ,QACxB,aAAY,IAAI,MAAM,OAAA,GAAA,0BAAA,oBAAyB,IAAI,CAAC;AAGtD,SAAO;GACL,MAAM;GACN,WAAW,QAAQ,MAAM,WAAW;GACpC,aAAa;GACd;GACD;AAEF,KAAI,OAAO,KACT,QAAO;EAAE,QAAQ,OAAO;EAAQ,aAAa,OAAO;EAAa;AAGnE,MAAK,aAAa;CAGlB,MAAM,EAAE,cAAc;AACtB,MAAK,IAAI,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK,YAAY;EACrD,MAAM,QAAQ,UAAU,MAAM,GAAG,IAAI,WAAW;AAChD,OAAK;AAEL,QAAM,GAAG,YAAY,wBAAwB,OAAO,OAAO;GACzD,MAAM,0BAAU,IAAI,KAA0B;AAE9C,oBAAiB,IAAI,UAAU,OAAO,aAAa,SADvC,KAAK,KAAK,CAAC,UAAU,CAC+B;AAChE,SAAM,GAAG,QAAQ;AAEjB,QAAK,MAAM,CAAC,MAAM,QAAQ,QACxB,aAAY,IAAI,MAAM,OAAA,GAAA,0BAAA,oBAAyB,IAAI,CAAC;IAEtD;;CAGJ,MAAM,SAAS,YAAY,IAAI,SAAS;AACxC,KAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,gDAAgD;AAC7E,QAAO;EAAE;EAAQ,aAAa,OAAO;EAAa;;;;;;;;AASpD,eAAsB,mBACpB,IACA,MACA,SAC2B;CAC3B,MAAM,OAAO,aAAa;CAC1B,MAAM,KAAK,YAAY,KAAK;AAE5B,KAAI;EAEF,IAAI;AACJ,UAAQ,KAAK,MAAb;GACE,KAAK;AACH,eAAA,GAAA,iCAAA,eAAwB,KAAK,QAAQ;AACrC;GACF,KAAK;AACH,cAAU,KAAK;AACf;GACF,KAAK,SACH,QAAO,KAAK;GACd,KAAK,gBACH,OAAM,IAAI,MACR,oFACD;GACH,QAEE,OAAM,IAAI,MAAM,yBADC,KACmC,OAAO;;AAI/D,OAAK,iBAAiB,QAAQ;EAG9B,MAAM,WAAW,YAAY,KAAK;EAClC,MAAM,QAAQ,oBAAoB,QAAQ;AAC1C,OAAK,YAAY,YAAY,KAAK,GAAG;AACrC,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,kCAAkC;AAE1E,OAAK,aAAa,MAAM;EACxB,MAAM,WAAW,MAAM,MAAM,SAAS,GAAG;EAGzC,MAAM,aAAa,YAAY,KAAK;EACpC,MAAM,WAAW,SAAS,mBAAoB,MAAM,yBAAyB,GAAG;AAChF,OAAK,cAAc,YAAY,KAAK,GAAG;EAIvC,MAAM,cAAc;AACpB,OAAK,IAAI,UAAU,GAAG,UAAU,aAAa,UAC3C,KAAI;GACF,MAAM,OAAO,YAAY,KAAK;GAC9B,MAAM,SAAS,MAAM,YAAY,IAAI,UAAU,UAAU,OAAO,KAAK;AACrE,QAAK,gBAAgB,YAAY,KAAK,GAAG;AACzC,QAAK,UAAU;AAGf,OAAI,OAAO,eAAA,IAAoC;AAC7C,UAAM,MAAM,IAAI,SAAS;AACzB,SAAK,cAAc;;AAGrB,UAAO,OAAO;WACP,GAAG;AACV,OAAI,YAAY,cAAc,EAAG,OAAM;AAEvC,QAAK,YAAY;AACjB,QAAK,cAAc;;AAIvB,QAAM,IAAI,MAAM,mBAAmB;WAC3B;AACR,OAAK,UAAU,YAAY,KAAK,GAAG;AACnC,MAAIC,cAAAA,eAAe,CAAC,qBAClB,SAAQ,IAAI,mBAAmB,KAAK,UAAU,KAAK,GAAG;;;;;;;;AAY5D,eAAsB,uBACpB,IACA,MACA,SACgC;AAChC,KAAI,KAAK,SAAS,SAAS,SAAU,QAAO;CAE5C,MAAM,aAAa,MAAM,mBAAmB,IAAI,KAAK,UAAU,QAAQ;AACvE,QAAO;EACL,GAAG;EACH,UAAU;GAAE,MAAM;GAAU;GAAY;EACzC"}
|
|
1
|
+
{"version":3,"file":"template_cache.cjs","names":["PlTemplateLibV1","PlTemplateSoftwareV1","PlWasmV1","PlTemplateV1","PlTemplateOverrideV1","getDebugFlags"],"sources":["../../../src/mutator/template/template_cache.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport type {\n AnyResourceRef,\n PlClient,\n PlTransaction,\n SignedResourceId,\n ResourceRef,\n} from \"@milaboratories/pl-client\";\nimport {\n ensureSignedResourceIdNotNull,\n field,\n resourceType,\n toGlobalResourceId,\n} from \"@milaboratories/pl-client\";\nimport {\n parseTemplate,\n PlTemplateLibV1,\n PlTemplateOverrideV1,\n PlTemplateSoftwareV1,\n PlTemplateV1,\n PlWasmV1,\n} from \"@milaboratories/pl-model-backend\";\nimport type {\n CompiledTemplateV3,\n TemplateData,\n TemplateDataV3,\n TemplateLibData,\n TemplateLibDataV3,\n TemplateSoftwareData,\n TemplateSoftwareDataV3,\n TemplateWasmDataV3,\n} from \"@milaboratories/pl-model-backend\";\nimport { notEmpty } from \"@milaboratories/ts-helpers\";\nimport type { BlockPackSpecPrepared } from \"../../model\";\nimport type { TemplateSpecPrepared } from \"../../model/template_spec\";\nimport { getDebugFlags } from \"../../debug\";\n\nexport const TemplateCacheType = resourceType(\"TemplateCache\", \"1\");\n\nexport const TemplateCacheFieldName = \"__templateCache\";\nconst BATCH_SIZE = 50;\n/** @internal exported for testing */\nexport const GC_ACCESS_THRESHOLD = 30;\n/** @internal exported for testing */\nexport const GC_MAX_ENTRIES = 3000;\n/** @internal exported for testing */\nexport const ACCESS_COUNT_KEY = \"_accessCount\";\n/** @internal exported for testing */\nexport const ACCESS_KEY_PREFIX = \"access_\";\n\n// ─── Stats ───────────────────────────────────────────────────────────────────\n\nexport type TemplateCacheStat = {\n totalMs: number;\n flattenMs: number;\n cacheInitMs: number;\n materializeMs: number;\n totalNodes: number;\n cacheHits: number;\n cacheMisses: number;\n batchCount: number;\n happyPath: boolean;\n gcTriggered: boolean;\n retries: number;\n templateFormat: string;\n};\n\nfunction initialStat(): TemplateCacheStat {\n return {\n totalMs: 0,\n flattenMs: 0,\n cacheInitMs: 0,\n materializeMs: 0,\n totalNodes: 0,\n cacheHits: 0,\n cacheMisses: 0,\n batchCount: 0,\n happyPath: false,\n gcTriggered: false,\n retries: 0,\n templateFormat: \"\",\n };\n}\n\n// ─── Tree node abstraction ───────────────────────────────────────────────────\n\ninterface CacheableNode {\n /** SHA-256 content hash (includes all descendant content) */\n hash: string;\n /** Creates this node's resource in a transaction.\n * childRefs maps child hash → already-resolved ResourceRef or SignedResourceId */\n create: (tx: PlTransaction, childRefs: ReadonlyMap<string, AnyResourceRef>) => ResourceRef;\n /** Hashes of direct child nodes this node depends on */\n childHashes: string[];\n}\n\n// ─── Hash computation helpers ────────────────────────────────────────────────\n\nfunction getSourceCode(name: string, sources: Record<string, string>, sourceHash: string): string {\n return notEmpty(\n sources[sourceHash],\n `trying to get \"${name}\" source: sources map doesn't contain source hash ${sourceHash}`,\n );\n}\n\n/**\n * Bottom-up hash composition: each node hashes its OWN content + child hash STRINGS.\n * This means each unique node is hashed exactly once → O(n) instead of O(n * depth).\n */\n\n// V2 leaf hashes (libs, software — no children)\n\nfunction hashLibV2(lib: TemplateLibData): string {\n return createHash(\"sha256\")\n .update(PlTemplateLibV1.type.name)\n .update(PlTemplateLibV1.type.version)\n .update(lib.name)\n .update(lib.version)\n .update(lib.src)\n .digest(\"hex\");\n}\n\nfunction hashSoftwareV2(sw: TemplateSoftwareData): string {\n return createHash(\"sha256\")\n .update(PlTemplateSoftwareV1.type.name)\n .update(PlTemplateSoftwareV1.type.version)\n .update(sw.name)\n .update(sw.version)\n .update(sw.src)\n .digest(\"hex\");\n}\n\n// V3 leaf hashes — use sourceHash directly instead of resolving source content\n\nfunction hashLibV3(lib: TemplateLibDataV3): string {\n return createHash(\"sha256\")\n .update(PlTemplateLibV1.type.name)\n .update(PlTemplateLibV1.type.version)\n .update(lib.name)\n .update(lib.version)\n .update(lib.sourceHash)\n .digest(\"hex\");\n}\n\nfunction hashSoftwareV3(sw: TemplateSoftwareDataV3): string {\n return createHash(\"sha256\")\n .update(PlTemplateSoftwareV1.type.name)\n .update(PlTemplateSoftwareV1.type.version)\n .update(sw.name)\n .update(sw.version)\n .update(sw.sourceHash)\n .digest(\"hex\");\n}\n\nfunction hashWasmV3(wasm: TemplateWasmDataV3): string {\n return createHash(\"sha256\")\n .update(PlWasmV1.type.name)\n .update(PlWasmV1.type.version)\n .update(wasm.name)\n .update(wasm.version)\n .update(wasm.sourceHash)\n .digest(\"hex\");\n}\n\n// ─── Tree flattening ─────────────────────────────────────────────────────────\n\nfunction flattenV2Tree(data: TemplateData): CacheableNode[] {\n const nodes: CacheableNode[] = [];\n const seen = new Set<string>();\n\n function processLib(lib: TemplateLibData): string {\n const hash = hashLibV2(lib);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) =>\n tx.createValue(\n PlTemplateLibV1.type,\n JSON.stringify(PlTemplateLibV1.fromV2Data(lib).data),\n ),\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processSoftware(sw: TemplateSoftwareData): string {\n const hash = hashSoftwareV2(sw);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) => {\n const swData = PlTemplateSoftwareV1.fromV2Data(sw);\n const ref = tx.createStruct(PlTemplateSoftwareV1.type, swData.data);\n tx.setKValue(ref, PlTemplateSoftwareV1.metaNameKey, JSON.stringify(swData.name));\n tx.lock(ref);\n return ref;\n },\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processTemplate(tpl: TemplateData): string {\n // Process children first (bottom-up) — their hashes are computed before ours\n const childHashes: string[] = [];\n const children: { fieldName: string; hash: string }[] = [];\n\n for (const [libId, lib] of Object.entries(tpl.libs ?? {})) {\n const h = processLib(lib);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.libPrefix}/${libId}`, hash: h });\n }\n for (const [swId, sw] of Object.entries(tpl.software ?? {})) {\n const h = processSoftware(sw);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.softPrefix}/${swId}`, hash: h });\n }\n for (const [swId, sw] of Object.entries(tpl.assets ?? {})) {\n const h = processSoftware(sw);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.softPrefix}/${swId}`, hash: h });\n }\n for (const [tplId, sub] of Object.entries(tpl.templates ?? {})) {\n const h = processTemplate(sub);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.tplPrefix}/${tplId}`, hash: h });\n }\n\n // Compose hash from own content + child hash strings (NOT child content)\n const h = createHash(\"sha256\")\n .update(PlTemplateV1.type.name)\n .update(PlTemplateV1.type.version)\n .update(tpl.hashOverride ?? \"no-override\")\n .update(tpl.name)\n .update(tpl.version)\n .update(tpl.src);\n for (const child of children) {\n h.update(\"child:\" + child.fieldName + \":\" + child.hash);\n }\n const hash = h.digest(\"hex\");\n\n if (seen.has(hash)) return hash;\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx, childRefs) => {\n const tplRef = tx.createStruct(\n PlTemplateV1.type,\n JSON.stringify(PlTemplateV1.fromV2Data(tpl).data),\n );\n for (const child of children) {\n const fld = field(tplRef, child.fieldName);\n tx.createField(fld, \"Input\");\n tx.setField(fld, notEmpty(childRefs.get(child.hash), `missing child ref ${child.hash}`));\n }\n tx.lock(tplRef);\n\n if (!tpl.hashOverride) return tplRef;\n\n const overrideRef = tx.createStruct(\n PlTemplateOverrideV1.type,\n JSON.stringify(PlTemplateOverrideV1.fromV2Data(tpl)),\n );\n const overrideFld = PlTemplateOverrideV1.tplField(overrideRef);\n tx.createField(overrideFld, \"Service\");\n tx.setField(overrideFld, tplRef);\n tx.lock(overrideRef);\n return overrideRef;\n },\n childHashes,\n });\n\n return hash;\n }\n\n processTemplate(data);\n return nodes;\n}\n\nfunction flattenV3Tree(data: CompiledTemplateV3): CacheableNode[] {\n const nodes: CacheableNode[] = [];\n const seen = new Set<string>();\n const sources = data.hashToSource;\n\n function processLib(lib: TemplateLibDataV3): string {\n const hash = hashLibV3(lib);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) =>\n tx.createValue(\n PlTemplateLibV1.type,\n JSON.stringify(\n PlTemplateLibV1.fromV3Data(lib, getSourceCode(lib.name, sources, lib.sourceHash))\n .data,\n ),\n ),\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processSoftware(sw: TemplateSoftwareDataV3): string {\n const hash = hashSoftwareV3(sw);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) => {\n const swData = PlTemplateSoftwareV1.fromV3Data(\n sw,\n getSourceCode(sw.name, sources, sw.sourceHash),\n );\n const ref = tx.createStruct(PlTemplateSoftwareV1.type, swData.data);\n tx.setKValue(ref, PlTemplateSoftwareV1.metaNameKey, JSON.stringify(swData.name));\n tx.lock(ref);\n return ref;\n },\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processWasm(wasm: TemplateWasmDataV3): string {\n const hash = hashWasmV3(wasm);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) =>\n tx.createValue(\n PlWasmV1.type,\n JSON.stringify(\n PlWasmV1.fromV3Data(wasm, getSourceCode(wasm.name, sources, wasm.sourceHash)).data,\n ),\n ),\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processTemplate(tpl: TemplateDataV3): string {\n // Process children first (bottom-up)\n const childHashes: string[] = [];\n const children: { fieldName: string; hash: string }[] = [];\n\n for (const [libId, lib] of Object.entries(tpl.libs ?? {})) {\n const h = processLib(lib);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.libPrefix}/${libId}`, hash: h });\n }\n for (const [swId, sw] of Object.entries(tpl.software ?? {})) {\n const h = processSoftware(sw);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.softPrefix}/${swId}`, hash: h });\n }\n for (const [swId, sw] of Object.entries(tpl.assets ?? {})) {\n const h = processSoftware(sw);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.softPrefix}/${swId}`, hash: h });\n }\n for (const [tplId, sub] of Object.entries(tpl.templates ?? {})) {\n const h = processTemplate(sub);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.tplPrefix}/${tplId}`, hash: h });\n }\n for (const [wasmId, wasm] of Object.entries(tpl.wasm ?? {})) {\n const h = processWasm(wasm);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.wasmPrefix}/${wasmId}`, hash: h });\n }\n\n // Compose hash from own content + child hash strings (NOT child content).\n // Uses sourceHash directly — it already uniquely identifies the source.\n const h = createHash(\"sha256\")\n .update(PlTemplateV1.type.name)\n .update(PlTemplateV1.type.version)\n .update(tpl.hashOverride ?? \"no-override\")\n .update(tpl.name)\n .update(tpl.version)\n .update(tpl.sourceHash);\n for (const child of children) {\n h.update(\"child:\" + child.fieldName + \":\" + child.hash);\n }\n const hash = h.digest(\"hex\");\n\n if (seen.has(hash)) return hash;\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx, childRefs) => {\n const sourceCode = getSourceCode(tpl.name, sources, tpl.sourceHash);\n const tplRef = tx.createStruct(\n PlTemplateV1.type,\n JSON.stringify(PlTemplateV1.fromV3Data(tpl, sourceCode).data),\n );\n for (const child of children) {\n const fld = field(tplRef, child.fieldName);\n tx.createField(fld, \"Input\");\n tx.setField(fld, notEmpty(childRefs.get(child.hash), `missing child ref ${child.hash}`));\n }\n tx.lock(tplRef);\n\n if (!tpl.hashOverride) return tplRef;\n\n const overrideRef = tx.createStruct(\n PlTemplateOverrideV1.type,\n JSON.stringify(PlTemplateOverrideV1.fromV3Data(tpl)),\n );\n const overrideFld = PlTemplateOverrideV1.tplField(overrideRef);\n tx.createField(overrideFld, \"Service\");\n tx.setField(overrideFld, tplRef);\n tx.lock(overrideRef);\n return overrideRef;\n },\n childHashes,\n });\n\n return hash;\n }\n\n processTemplate(data.template);\n return nodes;\n}\n\n/** Flatten template tree into a topologically ordered list of cacheable nodes (leaves first). */\nexport function flattenTemplateTree(data: TemplateData | CompiledTemplateV3): CacheableNode[] {\n if (data.type === \"pl.tengo-template.v2\") {\n return flattenV2Tree(data);\n } else {\n return flattenV3Tree(data);\n }\n}\n\n// ─── Cache operations ────────────────────────────────────────────────────────\n\n/** In-memory cache for the TemplateCache SignedResourceId per PlClient instance. */\nconst cacheRidMap = new WeakMap<PlClient, SignedResourceId>();\n\n/** Clear the in-memory cacheRid entry (call on errors referencing the cache resource). */\nexport function invalidateTemplateCacheId(pl: PlClient): void {\n cacheRidMap.delete(pl);\n}\n\n/** Find or create the TemplateCache/1 resource on user root. */\nexport async function getOrCreateTemplateCache(pl: PlClient): Promise<SignedResourceId> {\n // Check in-memory cache first (0ms after first call)\n const cached = cacheRidMap.get(pl);\n if (cached !== undefined) return cached;\n\n // Try read-only check\n const existing = await pl.withReadTx(\"templateCache:check\", async (tx) => {\n const fd = await tx.getFieldIfExists(field(pl.clientRoot, TemplateCacheFieldName));\n return fd ? ensureSignedResourceIdNotNull(fd.value) : undefined;\n });\n if (existing) {\n cacheRidMap.set(pl, existing);\n return existing;\n }\n\n const result = await pl.withWriteTx(\"templateCache:init\", async (tx) => {\n // Double-check inside write tx (another instance may have created it)\n const fd = await tx.getFieldIfExists(field(pl.clientRoot, TemplateCacheFieldName));\n if (fd) return ensureSignedResourceIdNotNull(fd.value);\n\n const cache = tx.createStruct(TemplateCacheType);\n tx.createField(field(pl.clientRoot, TemplateCacheFieldName), \"Dynamic\", cache);\n tx.lock(cache);\n await tx.commit();\n return await cache.globalId;\n });\n cacheRidMap.set(pl, result);\n return result;\n}\n\n/** Remove the template cache from user root. */\nexport async function dropTemplateCache(pl: PlClient): Promise<void> {\n await pl.withWriteTx(\"templateCache:drop\", async (tx) => {\n const cacheField = field(pl.clientRoot, TemplateCacheFieldName);\n const fd = await tx.getFieldIfExists(cacheField);\n if (fd) {\n tx.removeField(cacheField);\n await tx.commit();\n }\n });\n invalidateTemplateCacheId(pl);\n}\n\n// ─── GC ──────────────────────────────────────────────────────────────────────\n\n/**\n * Run count-based garbage collection on the template cache.\n * Evicts least-recently-used entries when the cache exceeds maxEntries.\n * Always resets the access counter to 0.\n *\n * @internal exported for testing (maxEntries parameter allows low thresholds in tests)\n * @returns true if entries were evicted\n */\nexport async function runGc(\n pl: PlClient,\n cacheRid: SignedResourceId,\n maxEntries: number = GC_MAX_ENTRIES,\n): Promise<boolean> {\n return await pl.withWriteTx(\"templateCache:gc\", async (tx) => {\n const kvs = await tx.listKeyValuesString(cacheRid);\n const entries: { hash: string; timestamp: number }[] = [];\n for (const { key, value } of kvs) {\n if (!key.startsWith(ACCESS_KEY_PREFIX)) continue;\n entries.push({\n hash: key.slice(ACCESS_KEY_PREFIX.length),\n timestamp: parseInt(value, 10),\n });\n }\n\n // Always reset counter\n tx.setKValue(cacheRid, ACCESS_COUNT_KEY, \"0\");\n\n if (entries.length <= maxEntries) {\n await tx.commit();\n return false;\n }\n\n // Sort oldest first, evict until under limit\n entries.sort((a, b) => a.timestamp - b.timestamp);\n const toEvict = entries.length - maxEntries;\n for (let i = 0; i < toEvict; i++) {\n tx.removeField(field(cacheRid, entries[i].hash));\n tx.deleteKValue(cacheRid, ACCESS_KEY_PREFIX + entries[i].hash);\n }\n\n await tx.commit();\n return true;\n });\n}\n\n// ─── Batched materialization ─────────────────────────────────────────────────\n\n/** Create a batch of cache nodes in the current transaction. */\nfunction createBatchNodes(\n tx: PlTransaction,\n cacheRid: SignedResourceId,\n batch: CacheableNode[],\n resolvedIds: ReadonlyMap<string, SignedResourceId>,\n newRefs: Map<string, ResourceRef>,\n now: string,\n): void {\n for (const node of batch) {\n const childRefs = new Map<string, AnyResourceRef>();\n for (const ch of node.childHashes) {\n const resolved = resolvedIds.get(ch) ?? newRefs.get(ch);\n if (resolved === undefined) {\n throw new Error(`BUG: child ${ch} not resolved`);\n }\n childRefs.set(ch, resolved);\n }\n const ref = node.create(tx, childRefs);\n newRefs.set(node.hash, ref);\n tx.createField(field(cacheRid, node.hash), \"Dynamic\", ref);\n tx.setKValue(cacheRid, ACCESS_KEY_PREFIX + node.hash, now);\n }\n}\n\n/**\n * Materialize a template tree via the cache using \"probe all + batched creation\".\n *\n * Phase 1 (single write tx):\n * - Check existence of ALL hashes in one roundtrip\n * - Happy path: if root cached, update access tracking and return\n * - Otherwise: fetch ResourceIds for all cache hits, create first batch of missing nodes\n *\n * Phase 2..N (one write tx per batch):\n * - Create remaining missing nodes in BATCH_SIZE chunks\n *\n * @returns root SignedResourceId and current access count (for GC decision)\n */\nasync function materialize(\n pl: PlClient,\n cacheRid: SignedResourceId,\n rootHash: string,\n nodes: CacheableNode[],\n stat: TemplateCacheStat,\n): Promise<{ rootId: SignedResourceId; accessCount: number }> {\n const allHashes = nodes.map((n) => n.hash);\n const resolvedIds = new Map<string, SignedResourceId>();\n\n // Phase 1: probe all + first batch\n const phase1 = await pl.withWriteTx(\"templateCache:materialize\", async (tx) => {\n // 1 roundtrip: check all hashes + read access count\n const [exists, countStr] = await Promise.all([\n Promise.all(allHashes.map((h) => tx.fieldExists(field(cacheRid, h)))),\n tx.getKValueStringIfExists(cacheRid, ACCESS_COUNT_KEY),\n ]);\n\n const prevCount = countStr ? parseInt(countStr, 10) : 0;\n const newCount = prevCount + 1;\n const now = Date.now().toString();\n const rootIdx = allHashes.length - 1;\n\n // Happy path: root already cached\n if (exists[rootIdx]) {\n const rootFd = await tx.getField(field(cacheRid, rootHash));\n const rootRid = ensureSignedResourceIdNotNull(rootFd.value);\n tx.setKValue(cacheRid, ACCESS_KEY_PREFIX + rootHash, now);\n tx.setKValue(cacheRid, ACCESS_COUNT_KEY, newCount.toString());\n await tx.commit();\n stat.happyPath = true;\n stat.cacheHits = stat.totalNodes;\n stat.batchCount = 1;\n return { done: true as const, rootId: rootRid, accessCount: newCount };\n }\n\n // Fetch ResourceIds for all cache hits (1 roundtrip)\n const hitIndices: number[] = [];\n for (let i = 0; i < allHashes.length; i++) {\n if (exists[i]) hitIndices.push(i);\n }\n\n if (hitIndices.length > 0) {\n const hitFields = await Promise.all(\n hitIndices.map((i) => tx.getField(field(cacheRid, allHashes[i]))),\n );\n for (let j = 0; j < hitIndices.length; j++) {\n resolvedIds.set(\n allHashes[hitIndices[j]],\n ensureSignedResourceIdNotNull(hitFields[j].value),\n );\n }\n }\n stat.cacheHits = hitIndices.length;\n\n // Missing nodes (topo order preserved from flatten)\n const missing = nodes.filter((n) => !resolvedIds.has(n.hash));\n stat.cacheMisses = missing.length;\n\n // Create first batch of missing nodes\n const firstBatch = missing.slice(0, BATCH_SIZE);\n const newRefs = new Map<string, ResourceRef>();\n createBatchNodes(tx, cacheRid, firstBatch, resolvedIds, newRefs, now);\n\n // Update access tracking for cache hits\n for (const i of hitIndices) {\n tx.setKValue(cacheRid, ACCESS_KEY_PREFIX + allHashes[i], now);\n }\n tx.setKValue(cacheRid, ACCESS_COUNT_KEY, newCount.toString());\n\n await tx.commit();\n\n // Resolve new refs to global IDs (after commit)\n for (const [hash, ref] of newRefs) {\n resolvedIds.set(hash, await toGlobalResourceId(ref));\n }\n\n return {\n done: false as const,\n remaining: missing.slice(BATCH_SIZE),\n accessCount: newCount,\n };\n });\n\n if (phase1.done) {\n return { rootId: phase1.rootId, accessCount: phase1.accessCount };\n }\n\n stat.batchCount = 1;\n\n // Phase 2+: remaining batches\n const { remaining } = phase1;\n for (let i = 0; i < remaining.length; i += BATCH_SIZE) {\n const batch = remaining.slice(i, i + BATCH_SIZE);\n stat.batchCount++;\n\n await pl.withWriteTx(\"templateCache:create\", async (tx) => {\n const newRefs = new Map<string, ResourceRef>();\n const now = Date.now().toString();\n createBatchNodes(tx, cacheRid, batch, resolvedIds, newRefs, now);\n await tx.commit();\n\n for (const [hash, ref] of newRefs) {\n resolvedIds.set(hash, await toGlobalResourceId(ref));\n }\n });\n }\n\n const rootId = resolvedIds.get(rootHash);\n if (!rootId) throw new Error(\"BUG: root hash not resolved after all batches\");\n return { rootId, accessCount: phase1.accessCount };\n}\n\n/**\n * Materialize a template tree via the cache.\n * Manages its own transactions internally — do NOT call inside an existing tx.\n *\n * @returns concrete SignedResourceId of the root template\n */\nexport async function loadTemplateCached(\n pl: PlClient,\n spec: TemplateSpecPrepared,\n options?: { cacheResourceId?: SignedResourceId },\n): Promise<SignedResourceId> {\n const stat = initialStat();\n const t0 = performance.now();\n\n try {\n // Parse to data if needed\n let tplData: TemplateData | CompiledTemplateV3;\n switch (spec.type) {\n case \"explicit\":\n tplData = parseTemplate(spec.content);\n break;\n case \"prepared\":\n tplData = spec.data;\n break;\n case \"cached\":\n return spec.resourceId;\n case \"from-registry\":\n throw new Error(\n \"loadTemplateCached does not support from-registry specs; use loadTemplate instead\",\n );\n default: {\n const _: never = spec;\n throw new Error(`unexpected spec type: ${(_ as any).type}`);\n }\n }\n\n stat.templateFormat = tplData.type;\n\n // Flatten to ordered nodes\n const tFlatten = performance.now();\n const nodes = flattenTemplateTree(tplData);\n stat.flattenMs = performance.now() - tFlatten;\n if (nodes.length === 0) throw new Error(\"template tree produced no nodes\");\n\n stat.totalNodes = nodes.length;\n const rootHash = nodes[nodes.length - 1].hash;\n\n // Resolve or create cache resource\n const tCacheInit = performance.now();\n const cacheRid = options?.cacheResourceId ?? (await getOrCreateTemplateCache(pl));\n stat.cacheInitMs = performance.now() - tCacheInit;\n\n // Retry loop: if a write tx fails (e.g. concurrent GC invalidated a cached resource),\n // restart materialization from scratch.\n const MAX_RETRIES = 3;\n for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {\n try {\n const tMat = performance.now();\n const result = await materialize(pl, cacheRid, rootHash, nodes, stat);\n stat.materializeMs = performance.now() - tMat;\n stat.retries = attempt;\n\n // GC in separate tx if access count exceeded threshold\n if (result.accessCount >= GC_ACCESS_THRESHOLD) {\n await runGc(pl, cacheRid);\n stat.gcTriggered = true;\n }\n\n return result.rootId;\n } catch (e) {\n if (attempt === MAX_RETRIES - 1) throw e;\n // Retry from scratch — previous batch results may reference GC'd resources\n stat.cacheHits = 0;\n stat.cacheMisses = 0;\n }\n }\n\n throw new Error(\"BUG: unreachable\");\n } finally {\n stat.totalMs = performance.now() - t0;\n if (getDebugFlags().logTemplateCacheStat) {\n console.log(`[templateCache] ${JSON.stringify(stat)}`);\n }\n }\n}\n\n// ─── Caller helper ───────────────────────────────────────────────────────────\n\n/**\n * Pre-materialize a block pack's template via cache.\n * Returns a copy of the spec with the template replaced by a cached reference.\n * If the template is already cached, returns the spec unchanged.\n */\nexport async function cacheBlockPackTemplate(\n pl: PlClient,\n spec: BlockPackSpecPrepared,\n options?: { cacheResourceId?: SignedResourceId },\n): Promise<BlockPackSpecPrepared> {\n if (spec.template.type === \"cached\") return spec;\n\n const resourceId = await loadTemplateCached(pl, spec.template, options);\n return {\n ...spec,\n template: { type: \"cached\", resourceId },\n };\n}\n"],"mappings":";;;;;;;AAqCA,MAAa,qBAAA,GAAA,0BAAA,cAAiC,iBAAiB,IAAI;AAEnE,MAAa,yBAAyB;AACtC,MAAM,aAAa;;AAEnB,MAAa,sBAAsB;;AAEnC,MAAa,iBAAiB;;AAE9B,MAAa,mBAAmB;;AAEhC,MAAa,oBAAoB;AAmBjC,SAAS,cAAiC;AACxC,QAAO;EACL,SAAS;EACT,WAAW;EACX,aAAa;EACb,eAAe;EACf,YAAY;EACZ,WAAW;EACX,aAAa;EACb,YAAY;EACZ,WAAW;EACX,aAAa;EACb,SAAS;EACT,gBAAgB;EACjB;;AAiBH,SAAS,cAAc,MAAc,SAAiC,YAA4B;AAChG,SAAA,GAAA,2BAAA,UACE,QAAQ,aACR,kBAAkB,KAAK,oDAAoD,aAC5E;;;;;;AAUH,SAAS,UAAU,KAA8B;AAC/C,SAAA,GAAA,YAAA,YAAkB,SAAS,CACxB,OAAOA,iCAAAA,gBAAgB,KAAK,KAAK,CACjC,OAAOA,iCAAAA,gBAAgB,KAAK,QAAQ,CACpC,OAAO,IAAI,KAAK,CAChB,OAAO,IAAI,QAAQ,CACnB,OAAO,IAAI,IAAI,CACf,OAAO,MAAM;;AAGlB,SAAS,eAAe,IAAkC;AACxD,SAAA,GAAA,YAAA,YAAkB,SAAS,CACxB,OAAOC,iCAAAA,qBAAqB,KAAK,KAAK,CACtC,OAAOA,iCAAAA,qBAAqB,KAAK,QAAQ,CACzC,OAAO,GAAG,KAAK,CACf,OAAO,GAAG,QAAQ,CAClB,OAAO,GAAG,IAAI,CACd,OAAO,MAAM;;AAKlB,SAAS,UAAU,KAAgC;AACjD,SAAA,GAAA,YAAA,YAAkB,SAAS,CACxB,OAAOD,iCAAAA,gBAAgB,KAAK,KAAK,CACjC,OAAOA,iCAAAA,gBAAgB,KAAK,QAAQ,CACpC,OAAO,IAAI,KAAK,CAChB,OAAO,IAAI,QAAQ,CACnB,OAAO,IAAI,WAAW,CACtB,OAAO,MAAM;;AAGlB,SAAS,eAAe,IAAoC;AAC1D,SAAA,GAAA,YAAA,YAAkB,SAAS,CACxB,OAAOC,iCAAAA,qBAAqB,KAAK,KAAK,CACtC,OAAOA,iCAAAA,qBAAqB,KAAK,QAAQ,CACzC,OAAO,GAAG,KAAK,CACf,OAAO,GAAG,QAAQ,CAClB,OAAO,GAAG,WAAW,CACrB,OAAO,MAAM;;AAGlB,SAAS,WAAW,MAAkC;AACpD,SAAA,GAAA,YAAA,YAAkB,SAAS,CACxB,OAAOC,iCAAAA,SAAS,KAAK,KAAK,CAC1B,OAAOA,iCAAAA,SAAS,KAAK,QAAQ,CAC7B,OAAO,KAAK,KAAK,CACjB,OAAO,KAAK,QAAQ,CACpB,OAAO,KAAK,WAAW,CACvB,OAAO,MAAM;;AAKlB,SAAS,cAAc,MAAqC;CAC1D,MAAM,QAAyB,EAAE;CACjC,MAAM,uBAAO,IAAI,KAAa;CAE9B,SAAS,WAAW,KAA8B;EAChD,MAAM,OAAO,UAAU,IAAI;AAC3B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OACP,GAAG,YACDF,iCAAAA,gBAAgB,MAChB,KAAK,UAAUA,iCAAAA,gBAAgB,WAAW,IAAI,CAAC,KAAK,CACrD;IACH,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,gBAAgB,IAAkC;EACzD,MAAM,OAAO,eAAe,GAAG;AAC/B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OAAO;KACd,MAAM,SAASC,iCAAAA,qBAAqB,WAAW,GAAG;KAClD,MAAM,MAAM,GAAG,aAAaA,iCAAAA,qBAAqB,MAAM,OAAO,KAAK;AACnE,QAAG,UAAU,KAAKA,iCAAAA,qBAAqB,aAAa,KAAK,UAAU,OAAO,KAAK,CAAC;AAChF,QAAG,KAAK,IAAI;AACZ,YAAO;;IAET,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,gBAAgB,KAA2B;EAElD,MAAM,cAAwB,EAAE;EAChC,MAAM,WAAkD,EAAE;AAE1D,OAAK,MAAM,CAAC,OAAO,QAAQ,OAAO,QAAQ,IAAI,QAAQ,EAAE,CAAC,EAAE;GACzD,MAAM,IAAI,WAAW,IAAI;AACzB,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAGE,iCAAAA,aAAa,UAAU,GAAG;IAAS,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,MAAM,OAAO,OAAO,QAAQ,IAAI,YAAY,EAAE,CAAC,EAAE;GAC3D,MAAM,IAAI,gBAAgB,GAAG;AAC7B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAGA,iCAAAA,aAAa,WAAW,GAAG;IAAQ,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,MAAM,OAAO,OAAO,QAAQ,IAAI,UAAU,EAAE,CAAC,EAAE;GACzD,MAAM,IAAI,gBAAgB,GAAG;AAC7B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAGA,iCAAAA,aAAa,WAAW,GAAG;IAAQ,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,OAAO,QAAQ,OAAO,QAAQ,IAAI,aAAa,EAAE,CAAC,EAAE;GAC9D,MAAM,IAAI,gBAAgB,IAAI;AAC9B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAGA,iCAAAA,aAAa,UAAU,GAAG;IAAS,MAAM;IAAG,CAAC;;EAI7E,MAAM,KAAA,GAAA,YAAA,YAAe,SAAS,CAC3B,OAAOA,iCAAAA,aAAa,KAAK,KAAK,CAC9B,OAAOA,iCAAAA,aAAa,KAAK,QAAQ,CACjC,OAAO,IAAI,gBAAgB,cAAc,CACzC,OAAO,IAAI,KAAK,CAChB,OAAO,IAAI,QAAQ,CACnB,OAAO,IAAI,IAAI;AAClB,OAAK,MAAM,SAAS,SAClB,GAAE,OAAO,WAAW,MAAM,YAAY,MAAM,MAAM,KAAK;EAEzD,MAAM,OAAO,EAAE,OAAO,MAAM;AAE5B,MAAI,KAAK,IAAI,KAAK,CAAE,QAAO;AAC3B,OAAK,IAAI,KAAK;AACd,QAAM,KAAK;GACT;GACA,SAAS,IAAI,cAAc;IACzB,MAAM,SAAS,GAAG,aAChBA,iCAAAA,aAAa,MACb,KAAK,UAAUA,iCAAAA,aAAa,WAAW,IAAI,CAAC,KAAK,CAClD;AACD,SAAK,MAAM,SAAS,UAAU;KAC5B,MAAM,OAAA,GAAA,0BAAA,OAAY,QAAQ,MAAM,UAAU;AAC1C,QAAG,YAAY,KAAK,QAAQ;AAC5B,QAAG,SAAS,MAAA,GAAA,2BAAA,UAAc,UAAU,IAAI,MAAM,KAAK,EAAE,qBAAqB,MAAM,OAAO,CAAC;;AAE1F,OAAG,KAAK,OAAO;AAEf,QAAI,CAAC,IAAI,aAAc,QAAO;IAE9B,MAAM,cAAc,GAAG,aACrBC,iCAAAA,qBAAqB,MACrB,KAAK,UAAUA,iCAAAA,qBAAqB,WAAW,IAAI,CAAC,CACrD;IACD,MAAM,cAAcA,iCAAAA,qBAAqB,SAAS,YAAY;AAC9D,OAAG,YAAY,aAAa,UAAU;AACtC,OAAG,SAAS,aAAa,OAAO;AAChC,OAAG,KAAK,YAAY;AACpB,WAAO;;GAET;GACD,CAAC;AAEF,SAAO;;AAGT,iBAAgB,KAAK;AACrB,QAAO;;AAGT,SAAS,cAAc,MAA2C;CAChE,MAAM,QAAyB,EAAE;CACjC,MAAM,uBAAO,IAAI,KAAa;CAC9B,MAAM,UAAU,KAAK;CAErB,SAAS,WAAW,KAAgC;EAClD,MAAM,OAAO,UAAU,IAAI;AAC3B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OACP,GAAG,YACDJ,iCAAAA,gBAAgB,MAChB,KAAK,UACHA,iCAAAA,gBAAgB,WAAW,KAAK,cAAc,IAAI,MAAM,SAAS,IAAI,WAAW,CAAC,CAC9E,KACJ,CACF;IACH,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,gBAAgB,IAAoC;EAC3D,MAAM,OAAO,eAAe,GAAG;AAC/B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OAAO;KACd,MAAM,SAASC,iCAAAA,qBAAqB,WAClC,IACA,cAAc,GAAG,MAAM,SAAS,GAAG,WAAW,CAC/C;KACD,MAAM,MAAM,GAAG,aAAaA,iCAAAA,qBAAqB,MAAM,OAAO,KAAK;AACnE,QAAG,UAAU,KAAKA,iCAAAA,qBAAqB,aAAa,KAAK,UAAU,OAAO,KAAK,CAAC;AAChF,QAAG,KAAK,IAAI;AACZ,YAAO;;IAET,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,YAAY,MAAkC;EACrD,MAAM,OAAO,WAAW,KAAK;AAC7B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OACP,GAAG,YACDC,iCAAAA,SAAS,MACT,KAAK,UACHA,iCAAAA,SAAS,WAAW,MAAM,cAAc,KAAK,MAAM,SAAS,KAAK,WAAW,CAAC,CAAC,KAC/E,CACF;IACH,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,gBAAgB,KAA6B;EAEpD,MAAM,cAAwB,EAAE;EAChC,MAAM,WAAkD,EAAE;AAE1D,OAAK,MAAM,CAAC,OAAO,QAAQ,OAAO,QAAQ,IAAI,QAAQ,EAAE,CAAC,EAAE;GACzD,MAAM,IAAI,WAAW,IAAI;AACzB,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAGC,iCAAAA,aAAa,UAAU,GAAG;IAAS,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,MAAM,OAAO,OAAO,QAAQ,IAAI,YAAY,EAAE,CAAC,EAAE;GAC3D,MAAM,IAAI,gBAAgB,GAAG;AAC7B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAGA,iCAAAA,aAAa,WAAW,GAAG;IAAQ,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,MAAM,OAAO,OAAO,QAAQ,IAAI,UAAU,EAAE,CAAC,EAAE;GACzD,MAAM,IAAI,gBAAgB,GAAG;AAC7B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAGA,iCAAAA,aAAa,WAAW,GAAG;IAAQ,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,OAAO,QAAQ,OAAO,QAAQ,IAAI,aAAa,EAAE,CAAC,EAAE;GAC9D,MAAM,IAAI,gBAAgB,IAAI;AAC9B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAGA,iCAAAA,aAAa,UAAU,GAAG;IAAS,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,QAAQ,SAAS,OAAO,QAAQ,IAAI,QAAQ,EAAE,CAAC,EAAE;GAC3D,MAAM,IAAI,YAAY,KAAK;AAC3B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAGA,iCAAAA,aAAa,WAAW,GAAG;IAAU,MAAM;IAAG,CAAC;;EAK/E,MAAM,KAAA,GAAA,YAAA,YAAe,SAAS,CAC3B,OAAOA,iCAAAA,aAAa,KAAK,KAAK,CAC9B,OAAOA,iCAAAA,aAAa,KAAK,QAAQ,CACjC,OAAO,IAAI,gBAAgB,cAAc,CACzC,OAAO,IAAI,KAAK,CAChB,OAAO,IAAI,QAAQ,CACnB,OAAO,IAAI,WAAW;AACzB,OAAK,MAAM,SAAS,SAClB,GAAE,OAAO,WAAW,MAAM,YAAY,MAAM,MAAM,KAAK;EAEzD,MAAM,OAAO,EAAE,OAAO,MAAM;AAE5B,MAAI,KAAK,IAAI,KAAK,CAAE,QAAO;AAC3B,OAAK,IAAI,KAAK;AACd,QAAM,KAAK;GACT;GACA,SAAS,IAAI,cAAc;IACzB,MAAM,aAAa,cAAc,IAAI,MAAM,SAAS,IAAI,WAAW;IACnE,MAAM,SAAS,GAAG,aAChBA,iCAAAA,aAAa,MACb,KAAK,UAAUA,iCAAAA,aAAa,WAAW,KAAK,WAAW,CAAC,KAAK,CAC9D;AACD,SAAK,MAAM,SAAS,UAAU;KAC5B,MAAM,OAAA,GAAA,0BAAA,OAAY,QAAQ,MAAM,UAAU;AAC1C,QAAG,YAAY,KAAK,QAAQ;AAC5B,QAAG,SAAS,MAAA,GAAA,2BAAA,UAAc,UAAU,IAAI,MAAM,KAAK,EAAE,qBAAqB,MAAM,OAAO,CAAC;;AAE1F,OAAG,KAAK,OAAO;AAEf,QAAI,CAAC,IAAI,aAAc,QAAO;IAE9B,MAAM,cAAc,GAAG,aACrBC,iCAAAA,qBAAqB,MACrB,KAAK,UAAUA,iCAAAA,qBAAqB,WAAW,IAAI,CAAC,CACrD;IACD,MAAM,cAAcA,iCAAAA,qBAAqB,SAAS,YAAY;AAC9D,OAAG,YAAY,aAAa,UAAU;AACtC,OAAG,SAAS,aAAa,OAAO;AAChC,OAAG,KAAK,YAAY;AACpB,WAAO;;GAET;GACD,CAAC;AAEF,SAAO;;AAGT,iBAAgB,KAAK,SAAS;AAC9B,QAAO;;;AAIT,SAAgB,oBAAoB,MAA0D;AAC5F,KAAI,KAAK,SAAS,uBAChB,QAAO,cAAc,KAAK;KAE1B,QAAO,cAAc,KAAK;;;AAO9B,MAAM,8BAAc,IAAI,SAAqC;;AAG7D,SAAgB,0BAA0B,IAAoB;AAC5D,aAAY,OAAO,GAAG;;;AAIxB,eAAsB,yBAAyB,IAAyC;CAEtF,MAAM,SAAS,YAAY,IAAI,GAAG;AAClC,KAAI,WAAW,KAAA,EAAW,QAAO;CAGjC,MAAM,WAAW,MAAM,GAAG,WAAW,uBAAuB,OAAO,OAAO;EACxE,MAAM,KAAK,MAAM,GAAG,kBAAA,GAAA,0BAAA,OAAuB,GAAG,YAAY,uBAAuB,CAAC;AAClF,SAAO,MAAA,GAAA,0BAAA,+BAAmC,GAAG,MAAM,GAAG,KAAA;GACtD;AACF,KAAI,UAAU;AACZ,cAAY,IAAI,IAAI,SAAS;AAC7B,SAAO;;CAGT,MAAM,SAAS,MAAM,GAAG,YAAY,sBAAsB,OAAO,OAAO;EAEtE,MAAM,KAAK,MAAM,GAAG,kBAAA,GAAA,0BAAA,OAAuB,GAAG,YAAY,uBAAuB,CAAC;AAClF,MAAI,GAAI,SAAA,GAAA,0BAAA,+BAAqC,GAAG,MAAM;EAEtD,MAAM,QAAQ,GAAG,aAAa,kBAAkB;AAChD,KAAG,aAAA,GAAA,0BAAA,OAAkB,GAAG,YAAY,uBAAuB,EAAE,WAAW,MAAM;AAC9E,KAAG,KAAK,MAAM;AACd,QAAM,GAAG,QAAQ;AACjB,SAAO,MAAM,MAAM;GACnB;AACF,aAAY,IAAI,IAAI,OAAO;AAC3B,QAAO;;;AAIT,eAAsB,kBAAkB,IAA6B;AACnE,OAAM,GAAG,YAAY,sBAAsB,OAAO,OAAO;EACvD,MAAM,cAAA,GAAA,0BAAA,OAAmB,GAAG,YAAY,uBAAuB;AAE/D,MADW,MAAM,GAAG,iBAAiB,WAAW,EACxC;AACN,MAAG,YAAY,WAAW;AAC1B,SAAM,GAAG,QAAQ;;GAEnB;AACF,2BAA0B,GAAG;;;;;;;;;;AAa/B,eAAsB,MACpB,IACA,UACA,aAAqB,gBACH;AAClB,QAAO,MAAM,GAAG,YAAY,oBAAoB,OAAO,OAAO;EAC5D,MAAM,MAAM,MAAM,GAAG,oBAAoB,SAAS;EAClD,MAAM,UAAiD,EAAE;AACzD,OAAK,MAAM,EAAE,KAAK,WAAW,KAAK;AAChC,OAAI,CAAC,IAAI,WAAA,UAA6B,CAAE;AACxC,WAAQ,KAAK;IACX,MAAM,IAAI,MAAM,EAAyB;IACzC,WAAW,SAAS,OAAO,GAAG;IAC/B,CAAC;;AAIJ,KAAG,UAAU,UAAU,kBAAkB,IAAI;AAE7C,MAAI,QAAQ,UAAU,YAAY;AAChC,SAAM,GAAG,QAAQ;AACjB,UAAO;;AAIT,UAAQ,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,UAAU;EACjD,MAAM,UAAU,QAAQ,SAAS;AACjC,OAAK,IAAI,IAAI,GAAG,IAAI,SAAS,KAAK;AAChC,MAAG,aAAA,GAAA,0BAAA,OAAkB,UAAU,QAAQ,GAAG,KAAK,CAAC;AAChD,MAAG,aAAa,UAAU,oBAAoB,QAAQ,GAAG,KAAK;;AAGhE,QAAM,GAAG,QAAQ;AACjB,SAAO;GACP;;;AAMJ,SAAS,iBACP,IACA,UACA,OACA,aACA,SACA,KACM;AACN,MAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,4BAAY,IAAI,KAA6B;AACnD,OAAK,MAAM,MAAM,KAAK,aAAa;GACjC,MAAM,WAAW,YAAY,IAAI,GAAG,IAAI,QAAQ,IAAI,GAAG;AACvD,OAAI,aAAa,KAAA,EACf,OAAM,IAAI,MAAM,cAAc,GAAG,eAAe;AAElD,aAAU,IAAI,IAAI,SAAS;;EAE7B,MAAM,MAAM,KAAK,OAAO,IAAI,UAAU;AACtC,UAAQ,IAAI,KAAK,MAAM,IAAI;AAC3B,KAAG,aAAA,GAAA,0BAAA,OAAkB,UAAU,KAAK,KAAK,EAAE,WAAW,IAAI;AAC1D,KAAG,UAAU,UAAU,oBAAoB,KAAK,MAAM,IAAI;;;;;;;;;;;;;;;;AAiB9D,eAAe,YACb,IACA,UACA,UACA,OACA,MAC4D;CAC5D,MAAM,YAAY,MAAM,KAAK,MAAM,EAAE,KAAK;CAC1C,MAAM,8BAAc,IAAI,KAA+B;CAGvD,MAAM,SAAS,MAAM,GAAG,YAAY,6BAA6B,OAAO,OAAO;EAE7E,MAAM,CAAC,QAAQ,YAAY,MAAM,QAAQ,IAAI,CAC3C,QAAQ,IAAI,UAAU,KAAK,MAAM,GAAG,aAAA,GAAA,0BAAA,OAAkB,UAAU,EAAE,CAAC,CAAC,CAAC,EACrE,GAAG,wBAAwB,UAAU,iBAAiB,CACvD,CAAC;EAGF,MAAM,YADY,WAAW,SAAS,UAAU,GAAG,GAAG,KACzB;EAC7B,MAAM,MAAM,KAAK,KAAK,CAAC,UAAU;AAIjC,MAAI,OAHY,UAAU,SAAS,IAGd;GAEnB,MAAM,WAAA,GAAA,0BAAA,gCADS,MAAM,GAAG,UAAA,GAAA,0BAAA,OAAe,UAAU,SAAS,CAAC,EACN,MAAM;AAC3D,MAAG,UAAU,UAAU,oBAAoB,UAAU,IAAI;AACzD,MAAG,UAAU,UAAU,kBAAkB,SAAS,UAAU,CAAC;AAC7D,SAAM,GAAG,QAAQ;AACjB,QAAK,YAAY;AACjB,QAAK,YAAY,KAAK;AACtB,QAAK,aAAa;AAClB,UAAO;IAAE,MAAM;IAAe,QAAQ;IAAS,aAAa;IAAU;;EAIxE,MAAM,aAAuB,EAAE;AAC/B,OAAK,IAAI,IAAI,GAAG,IAAI,UAAU,QAAQ,IACpC,KAAI,OAAO,GAAI,YAAW,KAAK,EAAE;AAGnC,MAAI,WAAW,SAAS,GAAG;GACzB,MAAM,YAAY,MAAM,QAAQ,IAC9B,WAAW,KAAK,MAAM,GAAG,UAAA,GAAA,0BAAA,OAAe,UAAU,UAAU,GAAG,CAAC,CAAC,CAClE;AACD,QAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,IACrC,aAAY,IACV,UAAU,WAAW,MAAA,GAAA,0BAAA,+BACS,UAAU,GAAG,MAAM,CAClD;;AAGL,OAAK,YAAY,WAAW;EAG5B,MAAM,UAAU,MAAM,QAAQ,MAAM,CAAC,YAAY,IAAI,EAAE,KAAK,CAAC;AAC7D,OAAK,cAAc,QAAQ;EAG3B,MAAM,aAAa,QAAQ,MAAM,GAAG,WAAW;EAC/C,MAAM,0BAAU,IAAI,KAA0B;AAC9C,mBAAiB,IAAI,UAAU,YAAY,aAAa,SAAS,IAAI;AAGrE,OAAK,MAAM,KAAK,WACd,IAAG,UAAU,UAAU,oBAAoB,UAAU,IAAI,IAAI;AAE/D,KAAG,UAAU,UAAU,kBAAkB,SAAS,UAAU,CAAC;AAE7D,QAAM,GAAG,QAAQ;AAGjB,OAAK,MAAM,CAAC,MAAM,QAAQ,QACxB,aAAY,IAAI,MAAM,OAAA,GAAA,0BAAA,oBAAyB,IAAI,CAAC;AAGtD,SAAO;GACL,MAAM;GACN,WAAW,QAAQ,MAAM,WAAW;GACpC,aAAa;GACd;GACD;AAEF,KAAI,OAAO,KACT,QAAO;EAAE,QAAQ,OAAO;EAAQ,aAAa,OAAO;EAAa;AAGnE,MAAK,aAAa;CAGlB,MAAM,EAAE,cAAc;AACtB,MAAK,IAAI,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK,YAAY;EACrD,MAAM,QAAQ,UAAU,MAAM,GAAG,IAAI,WAAW;AAChD,OAAK;AAEL,QAAM,GAAG,YAAY,wBAAwB,OAAO,OAAO;GACzD,MAAM,0BAAU,IAAI,KAA0B;AAE9C,oBAAiB,IAAI,UAAU,OAAO,aAAa,SADvC,KAAK,KAAK,CAAC,UAAU,CAC+B;AAChE,SAAM,GAAG,QAAQ;AAEjB,QAAK,MAAM,CAAC,MAAM,QAAQ,QACxB,aAAY,IAAI,MAAM,OAAA,GAAA,0BAAA,oBAAyB,IAAI,CAAC;IAEtD;;CAGJ,MAAM,SAAS,YAAY,IAAI,SAAS;AACxC,KAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,gDAAgD;AAC7E,QAAO;EAAE;EAAQ,aAAa,OAAO;EAAa;;;;;;;;AASpD,eAAsB,mBACpB,IACA,MACA,SAC2B;CAC3B,MAAM,OAAO,aAAa;CAC1B,MAAM,KAAK,YAAY,KAAK;AAE5B,KAAI;EAEF,IAAI;AACJ,UAAQ,KAAK,MAAb;GACE,KAAK;AACH,eAAA,GAAA,iCAAA,eAAwB,KAAK,QAAQ;AACrC;GACF,KAAK;AACH,cAAU,KAAK;AACf;GACF,KAAK,SACH,QAAO,KAAK;GACd,KAAK,gBACH,OAAM,IAAI,MACR,oFACD;GACH,QAEE,OAAM,IAAI,MAAM,yBADC,KACmC,OAAO;;AAI/D,OAAK,iBAAiB,QAAQ;EAG9B,MAAM,WAAW,YAAY,KAAK;EAClC,MAAM,QAAQ,oBAAoB,QAAQ;AAC1C,OAAK,YAAY,YAAY,KAAK,GAAG;AACrC,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,kCAAkC;AAE1E,OAAK,aAAa,MAAM;EACxB,MAAM,WAAW,MAAM,MAAM,SAAS,GAAG;EAGzC,MAAM,aAAa,YAAY,KAAK;EACpC,MAAM,WAAW,SAAS,mBAAoB,MAAM,yBAAyB,GAAG;AAChF,OAAK,cAAc,YAAY,KAAK,GAAG;EAIvC,MAAM,cAAc;AACpB,OAAK,IAAI,UAAU,GAAG,UAAU,aAAa,UAC3C,KAAI;GACF,MAAM,OAAO,YAAY,KAAK;GAC9B,MAAM,SAAS,MAAM,YAAY,IAAI,UAAU,UAAU,OAAO,KAAK;AACrE,QAAK,gBAAgB,YAAY,KAAK,GAAG;AACzC,QAAK,UAAU;AAGf,OAAI,OAAO,eAAA,IAAoC;AAC7C,UAAM,MAAM,IAAI,SAAS;AACzB,SAAK,cAAc;;AAGrB,UAAO,OAAO;WACP,GAAG;AACV,OAAI,YAAY,cAAc,EAAG,OAAM;AAEvC,QAAK,YAAY;AACjB,QAAK,cAAc;;AAIvB,QAAM,IAAI,MAAM,mBAAmB;WAC3B;AACR,OAAK,UAAU,YAAY,KAAK,GAAG;AACnC,MAAIC,cAAAA,eAAe,CAAC,qBAClB,SAAQ,IAAI,mBAAmB,KAAK,UAAU,KAAK,GAAG;;;;;;;;AAY5D,eAAsB,uBACpB,IACA,MACA,SACgC;AAChC,KAAI,KAAK,SAAS,SAAS,SAAU,QAAO;CAE5C,MAAM,aAAa,MAAM,mBAAmB,IAAI,KAAK,UAAU,QAAQ;AACvE,QAAO;EACL,GAAG;EACH,UAAU;GAAE,MAAM;GAAU;GAAY;EACzC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"template_cache.d.ts","names":[],"sources":["../../../src/mutator/template/template_cache.ts"],"mappings":";;;;;;;
|
|
1
|
+
{"version":3,"file":"template_cache.d.ts","names":[],"sources":["../../../src/mutator/template/template_cache.ts"],"mappings":";;;;;;;cAqCa,iBAAA,EAAsD,4BAAA,CAArC,YAAA;AAAA,cAEjB,sBAAA;;cAGA,mBAAA;AALb;AAAA,cAOa,cAAA;;cAEA,gBAAA;;cAEA,iBAAA;AAAA,KAID,iBAAA;EACV,OAAA;EACA,SAAA;EACA,WAAA;EACA,aAAA;EACA,UAAA;EACA,SAAA;EACA,WAAA;EACA,UAAA;EACA,SAAA;EACA,WAAA;EACA,OAAA;EACA,cAAA;AAAA;AAAA,UAsBQ,aAAA;EAxCmB;EA0C3B,IAAA;EA1C2B;;EA6C3B,MAAA,GAAS,EAAA,EAAI,aAAA,EAAe,SAAA,EAAW,WAAA,SAAoB,cAAA,MAAoB,WAAA;EA3CnD;EA6C5B,WAAA;AAAA;;iBAqVc,mBAAA,CAAoB,IAAA,EAAM,YAAA,GAAe,kBAAA,GAAqB,aAAA;;iBAc9D,yBAAA,CAA0B,EAAA,EAAI,QAAA;;iBAKxB,wBAAA,CAAyB,EAAA,EAAI,QAAA,GAAW,OAAA,CAAQ,gBAAA;;iBA+BhD,iBAAA,CAAkB,EAAA,EAAI,QAAA,GAAW,OAAA;;;;;;;;;iBAsBjC,KAAA,CACpB,EAAA,EAAI,QAAA,EACJ,QAAA,EAAU,gBAAA,EACV,UAAA,YACC,OAAA;;;AA7bD;;;;iBA6nBoB,kBAAA,CACpB,EAAA,EAAI,QAAA,EACJ,IAAA,EAAM,oBAAA,EACN,OAAA;EAAY,eAAA,GAAkB,gBAAA;AAAA,IAC7B,OAAA,CAAQ,gBAAA;;;;;;iBAmFW,sBAAA,CACpB,EAAA,EAAI,QAAA,EACJ,IAAA,EAAM,qBAAA,EACN,OAAA;EAAY,eAAA,GAAkB,gBAAA;AAAA,IAC7B,OAAA,CAAQ,qBAAA"}
|
|
@@ -2,7 +2,7 @@ import { getDebugFlags } from "../../debug/index.js";
|
|
|
2
2
|
import { notEmpty } from "@milaboratories/ts-helpers";
|
|
3
3
|
import { ensureSignedResourceIdNotNull, field, resourceType, toGlobalResourceId } from "@milaboratories/pl-client";
|
|
4
4
|
import { createHash } from "node:crypto";
|
|
5
|
-
import { PlTemplateLibV1, PlTemplateOverrideV1, PlTemplateSoftwareV1, PlTemplateV1, parseTemplate } from "@milaboratories/pl-model-backend";
|
|
5
|
+
import { PlTemplateLibV1, PlTemplateOverrideV1, PlTemplateSoftwareV1, PlTemplateV1, PlWasmV1, parseTemplate } from "@milaboratories/pl-model-backend";
|
|
6
6
|
//#region src/mutator/template/template_cache.ts
|
|
7
7
|
const TemplateCacheType = resourceType("TemplateCache", "1");
|
|
8
8
|
const TemplateCacheFieldName = "__templateCache";
|
|
@@ -50,6 +50,9 @@ function hashLibV3(lib) {
|
|
|
50
50
|
function hashSoftwareV3(sw) {
|
|
51
51
|
return createHash("sha256").update(PlTemplateSoftwareV1.type.name).update(PlTemplateSoftwareV1.type.version).update(sw.name).update(sw.version).update(sw.sourceHash).digest("hex");
|
|
52
52
|
}
|
|
53
|
+
function hashWasmV3(wasm) {
|
|
54
|
+
return createHash("sha256").update(PlWasmV1.type.name).update(PlWasmV1.type.version).update(wasm.name).update(wasm.version).update(wasm.sourceHash).digest("hex");
|
|
55
|
+
}
|
|
53
56
|
function flattenV2Tree(data) {
|
|
54
57
|
const nodes = [];
|
|
55
58
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -182,6 +185,18 @@ function flattenV3Tree(data) {
|
|
|
182
185
|
}
|
|
183
186
|
return hash;
|
|
184
187
|
}
|
|
188
|
+
function processWasm(wasm) {
|
|
189
|
+
const hash = hashWasmV3(wasm);
|
|
190
|
+
if (!seen.has(hash)) {
|
|
191
|
+
seen.add(hash);
|
|
192
|
+
nodes.push({
|
|
193
|
+
hash,
|
|
194
|
+
create: (tx) => tx.createValue(PlWasmV1.type, JSON.stringify(PlWasmV1.fromV3Data(wasm, getSourceCode(wasm.name, sources, wasm.sourceHash)).data)),
|
|
195
|
+
childHashes: []
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
return hash;
|
|
199
|
+
}
|
|
185
200
|
function processTemplate(tpl) {
|
|
186
201
|
const childHashes = [];
|
|
187
202
|
const children = [];
|
|
@@ -217,6 +232,14 @@ function flattenV3Tree(data) {
|
|
|
217
232
|
hash: h
|
|
218
233
|
});
|
|
219
234
|
}
|
|
235
|
+
for (const [wasmId, wasm] of Object.entries(tpl.wasm ?? {})) {
|
|
236
|
+
const h = processWasm(wasm);
|
|
237
|
+
childHashes.push(h);
|
|
238
|
+
children.push({
|
|
239
|
+
fieldName: `${PlTemplateV1.wasmPrefix}/${wasmId}`,
|
|
240
|
+
hash: h
|
|
241
|
+
});
|
|
242
|
+
}
|
|
220
243
|
const h = createHash("sha256").update(PlTemplateV1.type.name).update(PlTemplateV1.type.version).update(tpl.hashOverride ?? "no-override").update(tpl.name).update(tpl.version).update(tpl.sourceHash);
|
|
221
244
|
for (const child of children) h.update("child:" + child.fieldName + ":" + child.hash);
|
|
222
245
|
const hash = h.digest("hex");
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"template_cache.js","names":[],"sources":["../../../src/mutator/template/template_cache.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport type {\n AnyResourceRef,\n PlClient,\n PlTransaction,\n SignedResourceId,\n ResourceRef,\n} from \"@milaboratories/pl-client\";\nimport {\n ensureSignedResourceIdNotNull,\n field,\n resourceType,\n toGlobalResourceId,\n} from \"@milaboratories/pl-client\";\nimport {\n parseTemplate,\n PlTemplateLibV1,\n PlTemplateOverrideV1,\n PlTemplateSoftwareV1,\n PlTemplateV1,\n} from \"@milaboratories/pl-model-backend\";\nimport type {\n CompiledTemplateV3,\n TemplateData,\n TemplateDataV3,\n TemplateLibData,\n TemplateLibDataV3,\n TemplateSoftwareData,\n TemplateSoftwareDataV3,\n} from \"@milaboratories/pl-model-backend\";\nimport { notEmpty } from \"@milaboratories/ts-helpers\";\nimport type { BlockPackSpecPrepared } from \"../../model\";\nimport type { TemplateSpecPrepared } from \"../../model/template_spec\";\nimport { getDebugFlags } from \"../../debug\";\n\nexport const TemplateCacheType = resourceType(\"TemplateCache\", \"1\");\n\nexport const TemplateCacheFieldName = \"__templateCache\";\nconst BATCH_SIZE = 50;\n/** @internal exported for testing */\nexport const GC_ACCESS_THRESHOLD = 30;\n/** @internal exported for testing */\nexport const GC_MAX_ENTRIES = 3000;\n/** @internal exported for testing */\nexport const ACCESS_COUNT_KEY = \"_accessCount\";\n/** @internal exported for testing */\nexport const ACCESS_KEY_PREFIX = \"access_\";\n\n// ─── Stats ───────────────────────────────────────────────────────────────────\n\nexport type TemplateCacheStat = {\n totalMs: number;\n flattenMs: number;\n cacheInitMs: number;\n materializeMs: number;\n totalNodes: number;\n cacheHits: number;\n cacheMisses: number;\n batchCount: number;\n happyPath: boolean;\n gcTriggered: boolean;\n retries: number;\n templateFormat: string;\n};\n\nfunction initialStat(): TemplateCacheStat {\n return {\n totalMs: 0,\n flattenMs: 0,\n cacheInitMs: 0,\n materializeMs: 0,\n totalNodes: 0,\n cacheHits: 0,\n cacheMisses: 0,\n batchCount: 0,\n happyPath: false,\n gcTriggered: false,\n retries: 0,\n templateFormat: \"\",\n };\n}\n\n// ─── Tree node abstraction ───────────────────────────────────────────────────\n\ninterface CacheableNode {\n /** SHA-256 content hash (includes all descendant content) */\n hash: string;\n /** Creates this node's resource in a transaction.\n * childRefs maps child hash → already-resolved ResourceRef or SignedResourceId */\n create: (tx: PlTransaction, childRefs: ReadonlyMap<string, AnyResourceRef>) => ResourceRef;\n /** Hashes of direct child nodes this node depends on */\n childHashes: string[];\n}\n\n// ─── Hash computation helpers ────────────────────────────────────────────────\n\nfunction getSourceCode(name: string, sources: Record<string, string>, sourceHash: string): string {\n return notEmpty(\n sources[sourceHash],\n `trying to get \"${name}\" source: sources map doesn't contain source hash ${sourceHash}`,\n );\n}\n\n/**\n * Bottom-up hash composition: each node hashes its OWN content + child hash STRINGS.\n * This means each unique node is hashed exactly once → O(n) instead of O(n * depth).\n */\n\n// V2 leaf hashes (libs, software — no children)\n\nfunction hashLibV2(lib: TemplateLibData): string {\n return createHash(\"sha256\")\n .update(PlTemplateLibV1.type.name)\n .update(PlTemplateLibV1.type.version)\n .update(lib.name)\n .update(lib.version)\n .update(lib.src)\n .digest(\"hex\");\n}\n\nfunction hashSoftwareV2(sw: TemplateSoftwareData): string {\n return createHash(\"sha256\")\n .update(PlTemplateSoftwareV1.type.name)\n .update(PlTemplateSoftwareV1.type.version)\n .update(sw.name)\n .update(sw.version)\n .update(sw.src)\n .digest(\"hex\");\n}\n\n// V3 leaf hashes — use sourceHash directly instead of resolving source content\n\nfunction hashLibV3(lib: TemplateLibDataV3): string {\n return createHash(\"sha256\")\n .update(PlTemplateLibV1.type.name)\n .update(PlTemplateLibV1.type.version)\n .update(lib.name)\n .update(lib.version)\n .update(lib.sourceHash)\n .digest(\"hex\");\n}\n\nfunction hashSoftwareV3(sw: TemplateSoftwareDataV3): string {\n return createHash(\"sha256\")\n .update(PlTemplateSoftwareV1.type.name)\n .update(PlTemplateSoftwareV1.type.version)\n .update(sw.name)\n .update(sw.version)\n .update(sw.sourceHash)\n .digest(\"hex\");\n}\n\n// ─── Tree flattening ─────────────────────────────────────────────────────────\n\nfunction flattenV2Tree(data: TemplateData): CacheableNode[] {\n const nodes: CacheableNode[] = [];\n const seen = new Set<string>();\n\n function processLib(lib: TemplateLibData): string {\n const hash = hashLibV2(lib);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) =>\n tx.createValue(\n PlTemplateLibV1.type,\n JSON.stringify(PlTemplateLibV1.fromV2Data(lib).data),\n ),\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processSoftware(sw: TemplateSoftwareData): string {\n const hash = hashSoftwareV2(sw);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) => {\n const swData = PlTemplateSoftwareV1.fromV2Data(sw);\n const ref = tx.createStruct(PlTemplateSoftwareV1.type, swData.data);\n tx.setKValue(ref, PlTemplateSoftwareV1.metaNameKey, JSON.stringify(swData.name));\n tx.lock(ref);\n return ref;\n },\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processTemplate(tpl: TemplateData): string {\n // Process children first (bottom-up) — their hashes are computed before ours\n const childHashes: string[] = [];\n const children: { fieldName: string; hash: string }[] = [];\n\n for (const [libId, lib] of Object.entries(tpl.libs ?? {})) {\n const h = processLib(lib);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.libPrefix}/${libId}`, hash: h });\n }\n for (const [swId, sw] of Object.entries(tpl.software ?? {})) {\n const h = processSoftware(sw);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.softPrefix}/${swId}`, hash: h });\n }\n for (const [swId, sw] of Object.entries(tpl.assets ?? {})) {\n const h = processSoftware(sw);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.softPrefix}/${swId}`, hash: h });\n }\n for (const [tplId, sub] of Object.entries(tpl.templates ?? {})) {\n const h = processTemplate(sub);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.tplPrefix}/${tplId}`, hash: h });\n }\n\n // Compose hash from own content + child hash strings (NOT child content)\n const h = createHash(\"sha256\")\n .update(PlTemplateV1.type.name)\n .update(PlTemplateV1.type.version)\n .update(tpl.hashOverride ?? \"no-override\")\n .update(tpl.name)\n .update(tpl.version)\n .update(tpl.src);\n for (const child of children) {\n h.update(\"child:\" + child.fieldName + \":\" + child.hash);\n }\n const hash = h.digest(\"hex\");\n\n if (seen.has(hash)) return hash;\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx, childRefs) => {\n const tplRef = tx.createStruct(\n PlTemplateV1.type,\n JSON.stringify(PlTemplateV1.fromV2Data(tpl).data),\n );\n for (const child of children) {\n const fld = field(tplRef, child.fieldName);\n tx.createField(fld, \"Input\");\n tx.setField(fld, notEmpty(childRefs.get(child.hash), `missing child ref ${child.hash}`));\n }\n tx.lock(tplRef);\n\n if (!tpl.hashOverride) return tplRef;\n\n const overrideRef = tx.createStruct(\n PlTemplateOverrideV1.type,\n JSON.stringify(PlTemplateOverrideV1.fromV2Data(tpl)),\n );\n const overrideFld = PlTemplateOverrideV1.tplField(overrideRef);\n tx.createField(overrideFld, \"Service\");\n tx.setField(overrideFld, tplRef);\n tx.lock(overrideRef);\n return overrideRef;\n },\n childHashes,\n });\n\n return hash;\n }\n\n processTemplate(data);\n return nodes;\n}\n\nfunction flattenV3Tree(data: CompiledTemplateV3): CacheableNode[] {\n const nodes: CacheableNode[] = [];\n const seen = new Set<string>();\n const sources = data.hashToSource;\n\n function processLib(lib: TemplateLibDataV3): string {\n const hash = hashLibV3(lib);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) =>\n tx.createValue(\n PlTemplateLibV1.type,\n JSON.stringify(\n PlTemplateLibV1.fromV3Data(lib, getSourceCode(lib.name, sources, lib.sourceHash))\n .data,\n ),\n ),\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processSoftware(sw: TemplateSoftwareDataV3): string {\n const hash = hashSoftwareV3(sw);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) => {\n const swData = PlTemplateSoftwareV1.fromV3Data(\n sw,\n getSourceCode(sw.name, sources, sw.sourceHash),\n );\n const ref = tx.createStruct(PlTemplateSoftwareV1.type, swData.data);\n tx.setKValue(ref, PlTemplateSoftwareV1.metaNameKey, JSON.stringify(swData.name));\n tx.lock(ref);\n return ref;\n },\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processTemplate(tpl: TemplateDataV3): string {\n // Process children first (bottom-up)\n const childHashes: string[] = [];\n const children: { fieldName: string; hash: string }[] = [];\n\n for (const [libId, lib] of Object.entries(tpl.libs ?? {})) {\n const h = processLib(lib);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.libPrefix}/${libId}`, hash: h });\n }\n for (const [swId, sw] of Object.entries(tpl.software ?? {})) {\n const h = processSoftware(sw);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.softPrefix}/${swId}`, hash: h });\n }\n for (const [swId, sw] of Object.entries(tpl.assets ?? {})) {\n const h = processSoftware(sw);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.softPrefix}/${swId}`, hash: h });\n }\n for (const [tplId, sub] of Object.entries(tpl.templates ?? {})) {\n const h = processTemplate(sub);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.tplPrefix}/${tplId}`, hash: h });\n }\n\n // Compose hash from own content + child hash strings (NOT child content).\n // Uses sourceHash directly — it already uniquely identifies the source.\n const h = createHash(\"sha256\")\n .update(PlTemplateV1.type.name)\n .update(PlTemplateV1.type.version)\n .update(tpl.hashOverride ?? \"no-override\")\n .update(tpl.name)\n .update(tpl.version)\n .update(tpl.sourceHash);\n for (const child of children) {\n h.update(\"child:\" + child.fieldName + \":\" + child.hash);\n }\n const hash = h.digest(\"hex\");\n\n if (seen.has(hash)) return hash;\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx, childRefs) => {\n const sourceCode = getSourceCode(tpl.name, sources, tpl.sourceHash);\n const tplRef = tx.createStruct(\n PlTemplateV1.type,\n JSON.stringify(PlTemplateV1.fromV3Data(tpl, sourceCode).data),\n );\n for (const child of children) {\n const fld = field(tplRef, child.fieldName);\n tx.createField(fld, \"Input\");\n tx.setField(fld, notEmpty(childRefs.get(child.hash), `missing child ref ${child.hash}`));\n }\n tx.lock(tplRef);\n\n if (!tpl.hashOverride) return tplRef;\n\n const overrideRef = tx.createStruct(\n PlTemplateOverrideV1.type,\n JSON.stringify(PlTemplateOverrideV1.fromV3Data(tpl)),\n );\n const overrideFld = PlTemplateOverrideV1.tplField(overrideRef);\n tx.createField(overrideFld, \"Service\");\n tx.setField(overrideFld, tplRef);\n tx.lock(overrideRef);\n return overrideRef;\n },\n childHashes,\n });\n\n return hash;\n }\n\n processTemplate(data.template);\n return nodes;\n}\n\n/** Flatten template tree into a topologically ordered list of cacheable nodes (leaves first). */\nexport function flattenTemplateTree(data: TemplateData | CompiledTemplateV3): CacheableNode[] {\n if (data.type === \"pl.tengo-template.v2\") {\n return flattenV2Tree(data);\n } else {\n return flattenV3Tree(data);\n }\n}\n\n// ─── Cache operations ────────────────────────────────────────────────────────\n\n/** In-memory cache for the TemplateCache SignedResourceId per PlClient instance. */\nconst cacheRidMap = new WeakMap<PlClient, SignedResourceId>();\n\n/** Clear the in-memory cacheRid entry (call on errors referencing the cache resource). */\nexport function invalidateTemplateCacheId(pl: PlClient): void {\n cacheRidMap.delete(pl);\n}\n\n/** Find or create the TemplateCache/1 resource on user root. */\nexport async function getOrCreateTemplateCache(pl: PlClient): Promise<SignedResourceId> {\n // Check in-memory cache first (0ms after first call)\n const cached = cacheRidMap.get(pl);\n if (cached !== undefined) return cached;\n\n // Try read-only check\n const existing = await pl.withReadTx(\"templateCache:check\", async (tx) => {\n const fd = await tx.getFieldIfExists(field(pl.clientRoot, TemplateCacheFieldName));\n return fd ? ensureSignedResourceIdNotNull(fd.value) : undefined;\n });\n if (existing) {\n cacheRidMap.set(pl, existing);\n return existing;\n }\n\n const result = await pl.withWriteTx(\"templateCache:init\", async (tx) => {\n // Double-check inside write tx (another instance may have created it)\n const fd = await tx.getFieldIfExists(field(pl.clientRoot, TemplateCacheFieldName));\n if (fd) return ensureSignedResourceIdNotNull(fd.value);\n\n const cache = tx.createStruct(TemplateCacheType);\n tx.createField(field(pl.clientRoot, TemplateCacheFieldName), \"Dynamic\", cache);\n tx.lock(cache);\n await tx.commit();\n return await cache.globalId;\n });\n cacheRidMap.set(pl, result);\n return result;\n}\n\n/** Remove the template cache from user root. */\nexport async function dropTemplateCache(pl: PlClient): Promise<void> {\n await pl.withWriteTx(\"templateCache:drop\", async (tx) => {\n const cacheField = field(pl.clientRoot, TemplateCacheFieldName);\n const fd = await tx.getFieldIfExists(cacheField);\n if (fd) {\n tx.removeField(cacheField);\n await tx.commit();\n }\n });\n invalidateTemplateCacheId(pl);\n}\n\n// ─── GC ──────────────────────────────────────────────────────────────────────\n\n/**\n * Run count-based garbage collection on the template cache.\n * Evicts least-recently-used entries when the cache exceeds maxEntries.\n * Always resets the access counter to 0.\n *\n * @internal exported for testing (maxEntries parameter allows low thresholds in tests)\n * @returns true if entries were evicted\n */\nexport async function runGc(\n pl: PlClient,\n cacheRid: SignedResourceId,\n maxEntries: number = GC_MAX_ENTRIES,\n): Promise<boolean> {\n return await pl.withWriteTx(\"templateCache:gc\", async (tx) => {\n const kvs = await tx.listKeyValuesString(cacheRid);\n const entries: { hash: string; timestamp: number }[] = [];\n for (const { key, value } of kvs) {\n if (!key.startsWith(ACCESS_KEY_PREFIX)) continue;\n entries.push({\n hash: key.slice(ACCESS_KEY_PREFIX.length),\n timestamp: parseInt(value, 10),\n });\n }\n\n // Always reset counter\n tx.setKValue(cacheRid, ACCESS_COUNT_KEY, \"0\");\n\n if (entries.length <= maxEntries) {\n await tx.commit();\n return false;\n }\n\n // Sort oldest first, evict until under limit\n entries.sort((a, b) => a.timestamp - b.timestamp);\n const toEvict = entries.length - maxEntries;\n for (let i = 0; i < toEvict; i++) {\n tx.removeField(field(cacheRid, entries[i].hash));\n tx.deleteKValue(cacheRid, ACCESS_KEY_PREFIX + entries[i].hash);\n }\n\n await tx.commit();\n return true;\n });\n}\n\n// ─── Batched materialization ─────────────────────────────────────────────────\n\n/** Create a batch of cache nodes in the current transaction. */\nfunction createBatchNodes(\n tx: PlTransaction,\n cacheRid: SignedResourceId,\n batch: CacheableNode[],\n resolvedIds: ReadonlyMap<string, SignedResourceId>,\n newRefs: Map<string, ResourceRef>,\n now: string,\n): void {\n for (const node of batch) {\n const childRefs = new Map<string, AnyResourceRef>();\n for (const ch of node.childHashes) {\n const resolved = resolvedIds.get(ch) ?? newRefs.get(ch);\n if (resolved === undefined) {\n throw new Error(`BUG: child ${ch} not resolved`);\n }\n childRefs.set(ch, resolved);\n }\n const ref = node.create(tx, childRefs);\n newRefs.set(node.hash, ref);\n tx.createField(field(cacheRid, node.hash), \"Dynamic\", ref);\n tx.setKValue(cacheRid, ACCESS_KEY_PREFIX + node.hash, now);\n }\n}\n\n/**\n * Materialize a template tree via the cache using \"probe all + batched creation\".\n *\n * Phase 1 (single write tx):\n * - Check existence of ALL hashes in one roundtrip\n * - Happy path: if root cached, update access tracking and return\n * - Otherwise: fetch ResourceIds for all cache hits, create first batch of missing nodes\n *\n * Phase 2..N (one write tx per batch):\n * - Create remaining missing nodes in BATCH_SIZE chunks\n *\n * @returns root SignedResourceId and current access count (for GC decision)\n */\nasync function materialize(\n pl: PlClient,\n cacheRid: SignedResourceId,\n rootHash: string,\n nodes: CacheableNode[],\n stat: TemplateCacheStat,\n): Promise<{ rootId: SignedResourceId; accessCount: number }> {\n const allHashes = nodes.map((n) => n.hash);\n const resolvedIds = new Map<string, SignedResourceId>();\n\n // Phase 1: probe all + first batch\n const phase1 = await pl.withWriteTx(\"templateCache:materialize\", async (tx) => {\n // 1 roundtrip: check all hashes + read access count\n const [exists, countStr] = await Promise.all([\n Promise.all(allHashes.map((h) => tx.fieldExists(field(cacheRid, h)))),\n tx.getKValueStringIfExists(cacheRid, ACCESS_COUNT_KEY),\n ]);\n\n const prevCount = countStr ? parseInt(countStr, 10) : 0;\n const newCount = prevCount + 1;\n const now = Date.now().toString();\n const rootIdx = allHashes.length - 1;\n\n // Happy path: root already cached\n if (exists[rootIdx]) {\n const rootFd = await tx.getField(field(cacheRid, rootHash));\n const rootRid = ensureSignedResourceIdNotNull(rootFd.value);\n tx.setKValue(cacheRid, ACCESS_KEY_PREFIX + rootHash, now);\n tx.setKValue(cacheRid, ACCESS_COUNT_KEY, newCount.toString());\n await tx.commit();\n stat.happyPath = true;\n stat.cacheHits = stat.totalNodes;\n stat.batchCount = 1;\n return { done: true as const, rootId: rootRid, accessCount: newCount };\n }\n\n // Fetch ResourceIds for all cache hits (1 roundtrip)\n const hitIndices: number[] = [];\n for (let i = 0; i < allHashes.length; i++) {\n if (exists[i]) hitIndices.push(i);\n }\n\n if (hitIndices.length > 0) {\n const hitFields = await Promise.all(\n hitIndices.map((i) => tx.getField(field(cacheRid, allHashes[i]))),\n );\n for (let j = 0; j < hitIndices.length; j++) {\n resolvedIds.set(\n allHashes[hitIndices[j]],\n ensureSignedResourceIdNotNull(hitFields[j].value),\n );\n }\n }\n stat.cacheHits = hitIndices.length;\n\n // Missing nodes (topo order preserved from flatten)\n const missing = nodes.filter((n) => !resolvedIds.has(n.hash));\n stat.cacheMisses = missing.length;\n\n // Create first batch of missing nodes\n const firstBatch = missing.slice(0, BATCH_SIZE);\n const newRefs = new Map<string, ResourceRef>();\n createBatchNodes(tx, cacheRid, firstBatch, resolvedIds, newRefs, now);\n\n // Update access tracking for cache hits\n for (const i of hitIndices) {\n tx.setKValue(cacheRid, ACCESS_KEY_PREFIX + allHashes[i], now);\n }\n tx.setKValue(cacheRid, ACCESS_COUNT_KEY, newCount.toString());\n\n await tx.commit();\n\n // Resolve new refs to global IDs (after commit)\n for (const [hash, ref] of newRefs) {\n resolvedIds.set(hash, await toGlobalResourceId(ref));\n }\n\n return {\n done: false as const,\n remaining: missing.slice(BATCH_SIZE),\n accessCount: newCount,\n };\n });\n\n if (phase1.done) {\n return { rootId: phase1.rootId, accessCount: phase1.accessCount };\n }\n\n stat.batchCount = 1;\n\n // Phase 2+: remaining batches\n const { remaining } = phase1;\n for (let i = 0; i < remaining.length; i += BATCH_SIZE) {\n const batch = remaining.slice(i, i + BATCH_SIZE);\n stat.batchCount++;\n\n await pl.withWriteTx(\"templateCache:create\", async (tx) => {\n const newRefs = new Map<string, ResourceRef>();\n const now = Date.now().toString();\n createBatchNodes(tx, cacheRid, batch, resolvedIds, newRefs, now);\n await tx.commit();\n\n for (const [hash, ref] of newRefs) {\n resolvedIds.set(hash, await toGlobalResourceId(ref));\n }\n });\n }\n\n const rootId = resolvedIds.get(rootHash);\n if (!rootId) throw new Error(\"BUG: root hash not resolved after all batches\");\n return { rootId, accessCount: phase1.accessCount };\n}\n\n/**\n * Materialize a template tree via the cache.\n * Manages its own transactions internally — do NOT call inside an existing tx.\n *\n * @returns concrete SignedResourceId of the root template\n */\nexport async function loadTemplateCached(\n pl: PlClient,\n spec: TemplateSpecPrepared,\n options?: { cacheResourceId?: SignedResourceId },\n): Promise<SignedResourceId> {\n const stat = initialStat();\n const t0 = performance.now();\n\n try {\n // Parse to data if needed\n let tplData: TemplateData | CompiledTemplateV3;\n switch (spec.type) {\n case \"explicit\":\n tplData = parseTemplate(spec.content);\n break;\n case \"prepared\":\n tplData = spec.data;\n break;\n case \"cached\":\n return spec.resourceId;\n case \"from-registry\":\n throw new Error(\n \"loadTemplateCached does not support from-registry specs; use loadTemplate instead\",\n );\n default: {\n const _: never = spec;\n throw new Error(`unexpected spec type: ${(_ as any).type}`);\n }\n }\n\n stat.templateFormat = tplData.type;\n\n // Flatten to ordered nodes\n const tFlatten = performance.now();\n const nodes = flattenTemplateTree(tplData);\n stat.flattenMs = performance.now() - tFlatten;\n if (nodes.length === 0) throw new Error(\"template tree produced no nodes\");\n\n stat.totalNodes = nodes.length;\n const rootHash = nodes[nodes.length - 1].hash;\n\n // Resolve or create cache resource\n const tCacheInit = performance.now();\n const cacheRid = options?.cacheResourceId ?? (await getOrCreateTemplateCache(pl));\n stat.cacheInitMs = performance.now() - tCacheInit;\n\n // Retry loop: if a write tx fails (e.g. concurrent GC invalidated a cached resource),\n // restart materialization from scratch.\n const MAX_RETRIES = 3;\n for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {\n try {\n const tMat = performance.now();\n const result = await materialize(pl, cacheRid, rootHash, nodes, stat);\n stat.materializeMs = performance.now() - tMat;\n stat.retries = attempt;\n\n // GC in separate tx if access count exceeded threshold\n if (result.accessCount >= GC_ACCESS_THRESHOLD) {\n await runGc(pl, cacheRid);\n stat.gcTriggered = true;\n }\n\n return result.rootId;\n } catch (e) {\n if (attempt === MAX_RETRIES - 1) throw e;\n // Retry from scratch — previous batch results may reference GC'd resources\n stat.cacheHits = 0;\n stat.cacheMisses = 0;\n }\n }\n\n throw new Error(\"BUG: unreachable\");\n } finally {\n stat.totalMs = performance.now() - t0;\n if (getDebugFlags().logTemplateCacheStat) {\n console.log(`[templateCache] ${JSON.stringify(stat)}`);\n }\n }\n}\n\n// ─── Caller helper ───────────────────────────────────────────────────────────\n\n/**\n * Pre-materialize a block pack's template via cache.\n * Returns a copy of the spec with the template replaced by a cached reference.\n * If the template is already cached, returns the spec unchanged.\n */\nexport async function cacheBlockPackTemplate(\n pl: PlClient,\n spec: BlockPackSpecPrepared,\n options?: { cacheResourceId?: SignedResourceId },\n): Promise<BlockPackSpecPrepared> {\n if (spec.template.type === \"cached\") return spec;\n\n const resourceId = await loadTemplateCached(pl, spec.template, options);\n return {\n ...spec,\n template: { type: \"cached\", resourceId },\n };\n}\n"],"mappings":";;;;;;AAmCA,MAAa,oBAAoB,aAAa,iBAAiB,IAAI;AAEnE,MAAa,yBAAyB;AACtC,MAAM,aAAa;;AAEnB,MAAa,sBAAsB;;AAEnC,MAAa,iBAAiB;;AAE9B,MAAa,mBAAmB;;AAEhC,MAAa,oBAAoB;AAmBjC,SAAS,cAAiC;AACxC,QAAO;EACL,SAAS;EACT,WAAW;EACX,aAAa;EACb,eAAe;EACf,YAAY;EACZ,WAAW;EACX,aAAa;EACb,YAAY;EACZ,WAAW;EACX,aAAa;EACb,SAAS;EACT,gBAAgB;EACjB;;AAiBH,SAAS,cAAc,MAAc,SAAiC,YAA4B;AAChG,QAAO,SACL,QAAQ,aACR,kBAAkB,KAAK,oDAAoD,aAC5E;;;;;;AAUH,SAAS,UAAU,KAA8B;AAC/C,QAAO,WAAW,SAAS,CACxB,OAAO,gBAAgB,KAAK,KAAK,CACjC,OAAO,gBAAgB,KAAK,QAAQ,CACpC,OAAO,IAAI,KAAK,CAChB,OAAO,IAAI,QAAQ,CACnB,OAAO,IAAI,IAAI,CACf,OAAO,MAAM;;AAGlB,SAAS,eAAe,IAAkC;AACxD,QAAO,WAAW,SAAS,CACxB,OAAO,qBAAqB,KAAK,KAAK,CACtC,OAAO,qBAAqB,KAAK,QAAQ,CACzC,OAAO,GAAG,KAAK,CACf,OAAO,GAAG,QAAQ,CAClB,OAAO,GAAG,IAAI,CACd,OAAO,MAAM;;AAKlB,SAAS,UAAU,KAAgC;AACjD,QAAO,WAAW,SAAS,CACxB,OAAO,gBAAgB,KAAK,KAAK,CACjC,OAAO,gBAAgB,KAAK,QAAQ,CACpC,OAAO,IAAI,KAAK,CAChB,OAAO,IAAI,QAAQ,CACnB,OAAO,IAAI,WAAW,CACtB,OAAO,MAAM;;AAGlB,SAAS,eAAe,IAAoC;AAC1D,QAAO,WAAW,SAAS,CACxB,OAAO,qBAAqB,KAAK,KAAK,CACtC,OAAO,qBAAqB,KAAK,QAAQ,CACzC,OAAO,GAAG,KAAK,CACf,OAAO,GAAG,QAAQ,CAClB,OAAO,GAAG,WAAW,CACrB,OAAO,MAAM;;AAKlB,SAAS,cAAc,MAAqC;CAC1D,MAAM,QAAyB,EAAE;CACjC,MAAM,uBAAO,IAAI,KAAa;CAE9B,SAAS,WAAW,KAA8B;EAChD,MAAM,OAAO,UAAU,IAAI;AAC3B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OACP,GAAG,YACD,gBAAgB,MAChB,KAAK,UAAU,gBAAgB,WAAW,IAAI,CAAC,KAAK,CACrD;IACH,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,gBAAgB,IAAkC;EACzD,MAAM,OAAO,eAAe,GAAG;AAC/B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OAAO;KACd,MAAM,SAAS,qBAAqB,WAAW,GAAG;KAClD,MAAM,MAAM,GAAG,aAAa,qBAAqB,MAAM,OAAO,KAAK;AACnE,QAAG,UAAU,KAAK,qBAAqB,aAAa,KAAK,UAAU,OAAO,KAAK,CAAC;AAChF,QAAG,KAAK,IAAI;AACZ,YAAO;;IAET,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,gBAAgB,KAA2B;EAElD,MAAM,cAAwB,EAAE;EAChC,MAAM,WAAkD,EAAE;AAE1D,OAAK,MAAM,CAAC,OAAO,QAAQ,OAAO,QAAQ,IAAI,QAAQ,EAAE,CAAC,EAAE;GACzD,MAAM,IAAI,WAAW,IAAI;AACzB,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAG,aAAa,UAAU,GAAG;IAAS,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,MAAM,OAAO,OAAO,QAAQ,IAAI,YAAY,EAAE,CAAC,EAAE;GAC3D,MAAM,IAAI,gBAAgB,GAAG;AAC7B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAG,aAAa,WAAW,GAAG;IAAQ,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,MAAM,OAAO,OAAO,QAAQ,IAAI,UAAU,EAAE,CAAC,EAAE;GACzD,MAAM,IAAI,gBAAgB,GAAG;AAC7B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAG,aAAa,WAAW,GAAG;IAAQ,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,OAAO,QAAQ,OAAO,QAAQ,IAAI,aAAa,EAAE,CAAC,EAAE;GAC9D,MAAM,IAAI,gBAAgB,IAAI;AAC9B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAG,aAAa,UAAU,GAAG;IAAS,MAAM;IAAG,CAAC;;EAI7E,MAAM,IAAI,WAAW,SAAS,CAC3B,OAAO,aAAa,KAAK,KAAK,CAC9B,OAAO,aAAa,KAAK,QAAQ,CACjC,OAAO,IAAI,gBAAgB,cAAc,CACzC,OAAO,IAAI,KAAK,CAChB,OAAO,IAAI,QAAQ,CACnB,OAAO,IAAI,IAAI;AAClB,OAAK,MAAM,SAAS,SAClB,GAAE,OAAO,WAAW,MAAM,YAAY,MAAM,MAAM,KAAK;EAEzD,MAAM,OAAO,EAAE,OAAO,MAAM;AAE5B,MAAI,KAAK,IAAI,KAAK,CAAE,QAAO;AAC3B,OAAK,IAAI,KAAK;AACd,QAAM,KAAK;GACT;GACA,SAAS,IAAI,cAAc;IACzB,MAAM,SAAS,GAAG,aAChB,aAAa,MACb,KAAK,UAAU,aAAa,WAAW,IAAI,CAAC,KAAK,CAClD;AACD,SAAK,MAAM,SAAS,UAAU;KAC5B,MAAM,MAAM,MAAM,QAAQ,MAAM,UAAU;AAC1C,QAAG,YAAY,KAAK,QAAQ;AAC5B,QAAG,SAAS,KAAK,SAAS,UAAU,IAAI,MAAM,KAAK,EAAE,qBAAqB,MAAM,OAAO,CAAC;;AAE1F,OAAG,KAAK,OAAO;AAEf,QAAI,CAAC,IAAI,aAAc,QAAO;IAE9B,MAAM,cAAc,GAAG,aACrB,qBAAqB,MACrB,KAAK,UAAU,qBAAqB,WAAW,IAAI,CAAC,CACrD;IACD,MAAM,cAAc,qBAAqB,SAAS,YAAY;AAC9D,OAAG,YAAY,aAAa,UAAU;AACtC,OAAG,SAAS,aAAa,OAAO;AAChC,OAAG,KAAK,YAAY;AACpB,WAAO;;GAET;GACD,CAAC;AAEF,SAAO;;AAGT,iBAAgB,KAAK;AACrB,QAAO;;AAGT,SAAS,cAAc,MAA2C;CAChE,MAAM,QAAyB,EAAE;CACjC,MAAM,uBAAO,IAAI,KAAa;CAC9B,MAAM,UAAU,KAAK;CAErB,SAAS,WAAW,KAAgC;EAClD,MAAM,OAAO,UAAU,IAAI;AAC3B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OACP,GAAG,YACD,gBAAgB,MAChB,KAAK,UACH,gBAAgB,WAAW,KAAK,cAAc,IAAI,MAAM,SAAS,IAAI,WAAW,CAAC,CAC9E,KACJ,CACF;IACH,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,gBAAgB,IAAoC;EAC3D,MAAM,OAAO,eAAe,GAAG;AAC/B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OAAO;KACd,MAAM,SAAS,qBAAqB,WAClC,IACA,cAAc,GAAG,MAAM,SAAS,GAAG,WAAW,CAC/C;KACD,MAAM,MAAM,GAAG,aAAa,qBAAqB,MAAM,OAAO,KAAK;AACnE,QAAG,UAAU,KAAK,qBAAqB,aAAa,KAAK,UAAU,OAAO,KAAK,CAAC;AAChF,QAAG,KAAK,IAAI;AACZ,YAAO;;IAET,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,gBAAgB,KAA6B;EAEpD,MAAM,cAAwB,EAAE;EAChC,MAAM,WAAkD,EAAE;AAE1D,OAAK,MAAM,CAAC,OAAO,QAAQ,OAAO,QAAQ,IAAI,QAAQ,EAAE,CAAC,EAAE;GACzD,MAAM,IAAI,WAAW,IAAI;AACzB,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAG,aAAa,UAAU,GAAG;IAAS,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,MAAM,OAAO,OAAO,QAAQ,IAAI,YAAY,EAAE,CAAC,EAAE;GAC3D,MAAM,IAAI,gBAAgB,GAAG;AAC7B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAG,aAAa,WAAW,GAAG;IAAQ,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,MAAM,OAAO,OAAO,QAAQ,IAAI,UAAU,EAAE,CAAC,EAAE;GACzD,MAAM,IAAI,gBAAgB,GAAG;AAC7B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAG,aAAa,WAAW,GAAG;IAAQ,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,OAAO,QAAQ,OAAO,QAAQ,IAAI,aAAa,EAAE,CAAC,EAAE;GAC9D,MAAM,IAAI,gBAAgB,IAAI;AAC9B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAG,aAAa,UAAU,GAAG;IAAS,MAAM;IAAG,CAAC;;EAK7E,MAAM,IAAI,WAAW,SAAS,CAC3B,OAAO,aAAa,KAAK,KAAK,CAC9B,OAAO,aAAa,KAAK,QAAQ,CACjC,OAAO,IAAI,gBAAgB,cAAc,CACzC,OAAO,IAAI,KAAK,CAChB,OAAO,IAAI,QAAQ,CACnB,OAAO,IAAI,WAAW;AACzB,OAAK,MAAM,SAAS,SAClB,GAAE,OAAO,WAAW,MAAM,YAAY,MAAM,MAAM,KAAK;EAEzD,MAAM,OAAO,EAAE,OAAO,MAAM;AAE5B,MAAI,KAAK,IAAI,KAAK,CAAE,QAAO;AAC3B,OAAK,IAAI,KAAK;AACd,QAAM,KAAK;GACT;GACA,SAAS,IAAI,cAAc;IACzB,MAAM,aAAa,cAAc,IAAI,MAAM,SAAS,IAAI,WAAW;IACnE,MAAM,SAAS,GAAG,aAChB,aAAa,MACb,KAAK,UAAU,aAAa,WAAW,KAAK,WAAW,CAAC,KAAK,CAC9D;AACD,SAAK,MAAM,SAAS,UAAU;KAC5B,MAAM,MAAM,MAAM,QAAQ,MAAM,UAAU;AAC1C,QAAG,YAAY,KAAK,QAAQ;AAC5B,QAAG,SAAS,KAAK,SAAS,UAAU,IAAI,MAAM,KAAK,EAAE,qBAAqB,MAAM,OAAO,CAAC;;AAE1F,OAAG,KAAK,OAAO;AAEf,QAAI,CAAC,IAAI,aAAc,QAAO;IAE9B,MAAM,cAAc,GAAG,aACrB,qBAAqB,MACrB,KAAK,UAAU,qBAAqB,WAAW,IAAI,CAAC,CACrD;IACD,MAAM,cAAc,qBAAqB,SAAS,YAAY;AAC9D,OAAG,YAAY,aAAa,UAAU;AACtC,OAAG,SAAS,aAAa,OAAO;AAChC,OAAG,KAAK,YAAY;AACpB,WAAO;;GAET;GACD,CAAC;AAEF,SAAO;;AAGT,iBAAgB,KAAK,SAAS;AAC9B,QAAO;;;AAIT,SAAgB,oBAAoB,MAA0D;AAC5F,KAAI,KAAK,SAAS,uBAChB,QAAO,cAAc,KAAK;KAE1B,QAAO,cAAc,KAAK;;;AAO9B,MAAM,8BAAc,IAAI,SAAqC;;AAG7D,SAAgB,0BAA0B,IAAoB;AAC5D,aAAY,OAAO,GAAG;;;AAIxB,eAAsB,yBAAyB,IAAyC;CAEtF,MAAM,SAAS,YAAY,IAAI,GAAG;AAClC,KAAI,WAAW,KAAA,EAAW,QAAO;CAGjC,MAAM,WAAW,MAAM,GAAG,WAAW,uBAAuB,OAAO,OAAO;EACxE,MAAM,KAAK,MAAM,GAAG,iBAAiB,MAAM,GAAG,YAAY,uBAAuB,CAAC;AAClF,SAAO,KAAK,8BAA8B,GAAG,MAAM,GAAG,KAAA;GACtD;AACF,KAAI,UAAU;AACZ,cAAY,IAAI,IAAI,SAAS;AAC7B,SAAO;;CAGT,MAAM,SAAS,MAAM,GAAG,YAAY,sBAAsB,OAAO,OAAO;EAEtE,MAAM,KAAK,MAAM,GAAG,iBAAiB,MAAM,GAAG,YAAY,uBAAuB,CAAC;AAClF,MAAI,GAAI,QAAO,8BAA8B,GAAG,MAAM;EAEtD,MAAM,QAAQ,GAAG,aAAa,kBAAkB;AAChD,KAAG,YAAY,MAAM,GAAG,YAAY,uBAAuB,EAAE,WAAW,MAAM;AAC9E,KAAG,KAAK,MAAM;AACd,QAAM,GAAG,QAAQ;AACjB,SAAO,MAAM,MAAM;GACnB;AACF,aAAY,IAAI,IAAI,OAAO;AAC3B,QAAO;;;AAIT,eAAsB,kBAAkB,IAA6B;AACnE,OAAM,GAAG,YAAY,sBAAsB,OAAO,OAAO;EACvD,MAAM,aAAa,MAAM,GAAG,YAAY,uBAAuB;AAE/D,MADW,MAAM,GAAG,iBAAiB,WAAW,EACxC;AACN,MAAG,YAAY,WAAW;AAC1B,SAAM,GAAG,QAAQ;;GAEnB;AACF,2BAA0B,GAAG;;;;;;;;;;AAa/B,eAAsB,MACpB,IACA,UACA,aAAqB,gBACH;AAClB,QAAO,MAAM,GAAG,YAAY,oBAAoB,OAAO,OAAO;EAC5D,MAAM,MAAM,MAAM,GAAG,oBAAoB,SAAS;EAClD,MAAM,UAAiD,EAAE;AACzD,OAAK,MAAM,EAAE,KAAK,WAAW,KAAK;AAChC,OAAI,CAAC,IAAI,WAAA,UAA6B,CAAE;AACxC,WAAQ,KAAK;IACX,MAAM,IAAI,MAAM,EAAyB;IACzC,WAAW,SAAS,OAAO,GAAG;IAC/B,CAAC;;AAIJ,KAAG,UAAU,UAAU,kBAAkB,IAAI;AAE7C,MAAI,QAAQ,UAAU,YAAY;AAChC,SAAM,GAAG,QAAQ;AACjB,UAAO;;AAIT,UAAQ,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,UAAU;EACjD,MAAM,UAAU,QAAQ,SAAS;AACjC,OAAK,IAAI,IAAI,GAAG,IAAI,SAAS,KAAK;AAChC,MAAG,YAAY,MAAM,UAAU,QAAQ,GAAG,KAAK,CAAC;AAChD,MAAG,aAAa,UAAU,oBAAoB,QAAQ,GAAG,KAAK;;AAGhE,QAAM,GAAG,QAAQ;AACjB,SAAO;GACP;;;AAMJ,SAAS,iBACP,IACA,UACA,OACA,aACA,SACA,KACM;AACN,MAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,4BAAY,IAAI,KAA6B;AACnD,OAAK,MAAM,MAAM,KAAK,aAAa;GACjC,MAAM,WAAW,YAAY,IAAI,GAAG,IAAI,QAAQ,IAAI,GAAG;AACvD,OAAI,aAAa,KAAA,EACf,OAAM,IAAI,MAAM,cAAc,GAAG,eAAe;AAElD,aAAU,IAAI,IAAI,SAAS;;EAE7B,MAAM,MAAM,KAAK,OAAO,IAAI,UAAU;AACtC,UAAQ,IAAI,KAAK,MAAM,IAAI;AAC3B,KAAG,YAAY,MAAM,UAAU,KAAK,KAAK,EAAE,WAAW,IAAI;AAC1D,KAAG,UAAU,UAAU,oBAAoB,KAAK,MAAM,IAAI;;;;;;;;;;;;;;;;AAiB9D,eAAe,YACb,IACA,UACA,UACA,OACA,MAC4D;CAC5D,MAAM,YAAY,MAAM,KAAK,MAAM,EAAE,KAAK;CAC1C,MAAM,8BAAc,IAAI,KAA+B;CAGvD,MAAM,SAAS,MAAM,GAAG,YAAY,6BAA6B,OAAO,OAAO;EAE7E,MAAM,CAAC,QAAQ,YAAY,MAAM,QAAQ,IAAI,CAC3C,QAAQ,IAAI,UAAU,KAAK,MAAM,GAAG,YAAY,MAAM,UAAU,EAAE,CAAC,CAAC,CAAC,EACrE,GAAG,wBAAwB,UAAU,iBAAiB,CACvD,CAAC;EAGF,MAAM,YADY,WAAW,SAAS,UAAU,GAAG,GAAG,KACzB;EAC7B,MAAM,MAAM,KAAK,KAAK,CAAC,UAAU;AAIjC,MAAI,OAHY,UAAU,SAAS,IAGd;GAEnB,MAAM,UAAU,+BADD,MAAM,GAAG,SAAS,MAAM,UAAU,SAAS,CAAC,EACN,MAAM;AAC3D,MAAG,UAAU,UAAU,oBAAoB,UAAU,IAAI;AACzD,MAAG,UAAU,UAAU,kBAAkB,SAAS,UAAU,CAAC;AAC7D,SAAM,GAAG,QAAQ;AACjB,QAAK,YAAY;AACjB,QAAK,YAAY,KAAK;AACtB,QAAK,aAAa;AAClB,UAAO;IAAE,MAAM;IAAe,QAAQ;IAAS,aAAa;IAAU;;EAIxE,MAAM,aAAuB,EAAE;AAC/B,OAAK,IAAI,IAAI,GAAG,IAAI,UAAU,QAAQ,IACpC,KAAI,OAAO,GAAI,YAAW,KAAK,EAAE;AAGnC,MAAI,WAAW,SAAS,GAAG;GACzB,MAAM,YAAY,MAAM,QAAQ,IAC9B,WAAW,KAAK,MAAM,GAAG,SAAS,MAAM,UAAU,UAAU,GAAG,CAAC,CAAC,CAClE;AACD,QAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,IACrC,aAAY,IACV,UAAU,WAAW,KACrB,8BAA8B,UAAU,GAAG,MAAM,CAClD;;AAGL,OAAK,YAAY,WAAW;EAG5B,MAAM,UAAU,MAAM,QAAQ,MAAM,CAAC,YAAY,IAAI,EAAE,KAAK,CAAC;AAC7D,OAAK,cAAc,QAAQ;EAG3B,MAAM,aAAa,QAAQ,MAAM,GAAG,WAAW;EAC/C,MAAM,0BAAU,IAAI,KAA0B;AAC9C,mBAAiB,IAAI,UAAU,YAAY,aAAa,SAAS,IAAI;AAGrE,OAAK,MAAM,KAAK,WACd,IAAG,UAAU,UAAU,oBAAoB,UAAU,IAAI,IAAI;AAE/D,KAAG,UAAU,UAAU,kBAAkB,SAAS,UAAU,CAAC;AAE7D,QAAM,GAAG,QAAQ;AAGjB,OAAK,MAAM,CAAC,MAAM,QAAQ,QACxB,aAAY,IAAI,MAAM,MAAM,mBAAmB,IAAI,CAAC;AAGtD,SAAO;GACL,MAAM;GACN,WAAW,QAAQ,MAAM,WAAW;GACpC,aAAa;GACd;GACD;AAEF,KAAI,OAAO,KACT,QAAO;EAAE,QAAQ,OAAO;EAAQ,aAAa,OAAO;EAAa;AAGnE,MAAK,aAAa;CAGlB,MAAM,EAAE,cAAc;AACtB,MAAK,IAAI,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK,YAAY;EACrD,MAAM,QAAQ,UAAU,MAAM,GAAG,IAAI,WAAW;AAChD,OAAK;AAEL,QAAM,GAAG,YAAY,wBAAwB,OAAO,OAAO;GACzD,MAAM,0BAAU,IAAI,KAA0B;AAE9C,oBAAiB,IAAI,UAAU,OAAO,aAAa,SADvC,KAAK,KAAK,CAAC,UAAU,CAC+B;AAChE,SAAM,GAAG,QAAQ;AAEjB,QAAK,MAAM,CAAC,MAAM,QAAQ,QACxB,aAAY,IAAI,MAAM,MAAM,mBAAmB,IAAI,CAAC;IAEtD;;CAGJ,MAAM,SAAS,YAAY,IAAI,SAAS;AACxC,KAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,gDAAgD;AAC7E,QAAO;EAAE;EAAQ,aAAa,OAAO;EAAa;;;;;;;;AASpD,eAAsB,mBACpB,IACA,MACA,SAC2B;CAC3B,MAAM,OAAO,aAAa;CAC1B,MAAM,KAAK,YAAY,KAAK;AAE5B,KAAI;EAEF,IAAI;AACJ,UAAQ,KAAK,MAAb;GACE,KAAK;AACH,cAAU,cAAc,KAAK,QAAQ;AACrC;GACF,KAAK;AACH,cAAU,KAAK;AACf;GACF,KAAK,SACH,QAAO,KAAK;GACd,KAAK,gBACH,OAAM,IAAI,MACR,oFACD;GACH,QAEE,OAAM,IAAI,MAAM,yBADC,KACmC,OAAO;;AAI/D,OAAK,iBAAiB,QAAQ;EAG9B,MAAM,WAAW,YAAY,KAAK;EAClC,MAAM,QAAQ,oBAAoB,QAAQ;AAC1C,OAAK,YAAY,YAAY,KAAK,GAAG;AACrC,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,kCAAkC;AAE1E,OAAK,aAAa,MAAM;EACxB,MAAM,WAAW,MAAM,MAAM,SAAS,GAAG;EAGzC,MAAM,aAAa,YAAY,KAAK;EACpC,MAAM,WAAW,SAAS,mBAAoB,MAAM,yBAAyB,GAAG;AAChF,OAAK,cAAc,YAAY,KAAK,GAAG;EAIvC,MAAM,cAAc;AACpB,OAAK,IAAI,UAAU,GAAG,UAAU,aAAa,UAC3C,KAAI;GACF,MAAM,OAAO,YAAY,KAAK;GAC9B,MAAM,SAAS,MAAM,YAAY,IAAI,UAAU,UAAU,OAAO,KAAK;AACrE,QAAK,gBAAgB,YAAY,KAAK,GAAG;AACzC,QAAK,UAAU;AAGf,OAAI,OAAO,eAAA,IAAoC;AAC7C,UAAM,MAAM,IAAI,SAAS;AACzB,SAAK,cAAc;;AAGrB,UAAO,OAAO;WACP,GAAG;AACV,OAAI,YAAY,cAAc,EAAG,OAAM;AAEvC,QAAK,YAAY;AACjB,QAAK,cAAc;;AAIvB,QAAM,IAAI,MAAM,mBAAmB;WAC3B;AACR,OAAK,UAAU,YAAY,KAAK,GAAG;AACnC,MAAI,eAAe,CAAC,qBAClB,SAAQ,IAAI,mBAAmB,KAAK,UAAU,KAAK,GAAG;;;;;;;;AAY5D,eAAsB,uBACpB,IACA,MACA,SACgC;AAChC,KAAI,KAAK,SAAS,SAAS,SAAU,QAAO;CAE5C,MAAM,aAAa,MAAM,mBAAmB,IAAI,KAAK,UAAU,QAAQ;AACvE,QAAO;EACL,GAAG;EACH,UAAU;GAAE,MAAM;GAAU;GAAY;EACzC"}
|
|
1
|
+
{"version":3,"file":"template_cache.js","names":[],"sources":["../../../src/mutator/template/template_cache.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport type {\n AnyResourceRef,\n PlClient,\n PlTransaction,\n SignedResourceId,\n ResourceRef,\n} from \"@milaboratories/pl-client\";\nimport {\n ensureSignedResourceIdNotNull,\n field,\n resourceType,\n toGlobalResourceId,\n} from \"@milaboratories/pl-client\";\nimport {\n parseTemplate,\n PlTemplateLibV1,\n PlTemplateOverrideV1,\n PlTemplateSoftwareV1,\n PlTemplateV1,\n PlWasmV1,\n} from \"@milaboratories/pl-model-backend\";\nimport type {\n CompiledTemplateV3,\n TemplateData,\n TemplateDataV3,\n TemplateLibData,\n TemplateLibDataV3,\n TemplateSoftwareData,\n TemplateSoftwareDataV3,\n TemplateWasmDataV3,\n} from \"@milaboratories/pl-model-backend\";\nimport { notEmpty } from \"@milaboratories/ts-helpers\";\nimport type { BlockPackSpecPrepared } from \"../../model\";\nimport type { TemplateSpecPrepared } from \"../../model/template_spec\";\nimport { getDebugFlags } from \"../../debug\";\n\nexport const TemplateCacheType = resourceType(\"TemplateCache\", \"1\");\n\nexport const TemplateCacheFieldName = \"__templateCache\";\nconst BATCH_SIZE = 50;\n/** @internal exported for testing */\nexport const GC_ACCESS_THRESHOLD = 30;\n/** @internal exported for testing */\nexport const GC_MAX_ENTRIES = 3000;\n/** @internal exported for testing */\nexport const ACCESS_COUNT_KEY = \"_accessCount\";\n/** @internal exported for testing */\nexport const ACCESS_KEY_PREFIX = \"access_\";\n\n// ─── Stats ───────────────────────────────────────────────────────────────────\n\nexport type TemplateCacheStat = {\n totalMs: number;\n flattenMs: number;\n cacheInitMs: number;\n materializeMs: number;\n totalNodes: number;\n cacheHits: number;\n cacheMisses: number;\n batchCount: number;\n happyPath: boolean;\n gcTriggered: boolean;\n retries: number;\n templateFormat: string;\n};\n\nfunction initialStat(): TemplateCacheStat {\n return {\n totalMs: 0,\n flattenMs: 0,\n cacheInitMs: 0,\n materializeMs: 0,\n totalNodes: 0,\n cacheHits: 0,\n cacheMisses: 0,\n batchCount: 0,\n happyPath: false,\n gcTriggered: false,\n retries: 0,\n templateFormat: \"\",\n };\n}\n\n// ─── Tree node abstraction ───────────────────────────────────────────────────\n\ninterface CacheableNode {\n /** SHA-256 content hash (includes all descendant content) */\n hash: string;\n /** Creates this node's resource in a transaction.\n * childRefs maps child hash → already-resolved ResourceRef or SignedResourceId */\n create: (tx: PlTransaction, childRefs: ReadonlyMap<string, AnyResourceRef>) => ResourceRef;\n /** Hashes of direct child nodes this node depends on */\n childHashes: string[];\n}\n\n// ─── Hash computation helpers ────────────────────────────────────────────────\n\nfunction getSourceCode(name: string, sources: Record<string, string>, sourceHash: string): string {\n return notEmpty(\n sources[sourceHash],\n `trying to get \"${name}\" source: sources map doesn't contain source hash ${sourceHash}`,\n );\n}\n\n/**\n * Bottom-up hash composition: each node hashes its OWN content + child hash STRINGS.\n * This means each unique node is hashed exactly once → O(n) instead of O(n * depth).\n */\n\n// V2 leaf hashes (libs, software — no children)\n\nfunction hashLibV2(lib: TemplateLibData): string {\n return createHash(\"sha256\")\n .update(PlTemplateLibV1.type.name)\n .update(PlTemplateLibV1.type.version)\n .update(lib.name)\n .update(lib.version)\n .update(lib.src)\n .digest(\"hex\");\n}\n\nfunction hashSoftwareV2(sw: TemplateSoftwareData): string {\n return createHash(\"sha256\")\n .update(PlTemplateSoftwareV1.type.name)\n .update(PlTemplateSoftwareV1.type.version)\n .update(sw.name)\n .update(sw.version)\n .update(sw.src)\n .digest(\"hex\");\n}\n\n// V3 leaf hashes — use sourceHash directly instead of resolving source content\n\nfunction hashLibV3(lib: TemplateLibDataV3): string {\n return createHash(\"sha256\")\n .update(PlTemplateLibV1.type.name)\n .update(PlTemplateLibV1.type.version)\n .update(lib.name)\n .update(lib.version)\n .update(lib.sourceHash)\n .digest(\"hex\");\n}\n\nfunction hashSoftwareV3(sw: TemplateSoftwareDataV3): string {\n return createHash(\"sha256\")\n .update(PlTemplateSoftwareV1.type.name)\n .update(PlTemplateSoftwareV1.type.version)\n .update(sw.name)\n .update(sw.version)\n .update(sw.sourceHash)\n .digest(\"hex\");\n}\n\nfunction hashWasmV3(wasm: TemplateWasmDataV3): string {\n return createHash(\"sha256\")\n .update(PlWasmV1.type.name)\n .update(PlWasmV1.type.version)\n .update(wasm.name)\n .update(wasm.version)\n .update(wasm.sourceHash)\n .digest(\"hex\");\n}\n\n// ─── Tree flattening ─────────────────────────────────────────────────────────\n\nfunction flattenV2Tree(data: TemplateData): CacheableNode[] {\n const nodes: CacheableNode[] = [];\n const seen = new Set<string>();\n\n function processLib(lib: TemplateLibData): string {\n const hash = hashLibV2(lib);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) =>\n tx.createValue(\n PlTemplateLibV1.type,\n JSON.stringify(PlTemplateLibV1.fromV2Data(lib).data),\n ),\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processSoftware(sw: TemplateSoftwareData): string {\n const hash = hashSoftwareV2(sw);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) => {\n const swData = PlTemplateSoftwareV1.fromV2Data(sw);\n const ref = tx.createStruct(PlTemplateSoftwareV1.type, swData.data);\n tx.setKValue(ref, PlTemplateSoftwareV1.metaNameKey, JSON.stringify(swData.name));\n tx.lock(ref);\n return ref;\n },\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processTemplate(tpl: TemplateData): string {\n // Process children first (bottom-up) — their hashes are computed before ours\n const childHashes: string[] = [];\n const children: { fieldName: string; hash: string }[] = [];\n\n for (const [libId, lib] of Object.entries(tpl.libs ?? {})) {\n const h = processLib(lib);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.libPrefix}/${libId}`, hash: h });\n }\n for (const [swId, sw] of Object.entries(tpl.software ?? {})) {\n const h = processSoftware(sw);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.softPrefix}/${swId}`, hash: h });\n }\n for (const [swId, sw] of Object.entries(tpl.assets ?? {})) {\n const h = processSoftware(sw);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.softPrefix}/${swId}`, hash: h });\n }\n for (const [tplId, sub] of Object.entries(tpl.templates ?? {})) {\n const h = processTemplate(sub);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.tplPrefix}/${tplId}`, hash: h });\n }\n\n // Compose hash from own content + child hash strings (NOT child content)\n const h = createHash(\"sha256\")\n .update(PlTemplateV1.type.name)\n .update(PlTemplateV1.type.version)\n .update(tpl.hashOverride ?? \"no-override\")\n .update(tpl.name)\n .update(tpl.version)\n .update(tpl.src);\n for (const child of children) {\n h.update(\"child:\" + child.fieldName + \":\" + child.hash);\n }\n const hash = h.digest(\"hex\");\n\n if (seen.has(hash)) return hash;\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx, childRefs) => {\n const tplRef = tx.createStruct(\n PlTemplateV1.type,\n JSON.stringify(PlTemplateV1.fromV2Data(tpl).data),\n );\n for (const child of children) {\n const fld = field(tplRef, child.fieldName);\n tx.createField(fld, \"Input\");\n tx.setField(fld, notEmpty(childRefs.get(child.hash), `missing child ref ${child.hash}`));\n }\n tx.lock(tplRef);\n\n if (!tpl.hashOverride) return tplRef;\n\n const overrideRef = tx.createStruct(\n PlTemplateOverrideV1.type,\n JSON.stringify(PlTemplateOverrideV1.fromV2Data(tpl)),\n );\n const overrideFld = PlTemplateOverrideV1.tplField(overrideRef);\n tx.createField(overrideFld, \"Service\");\n tx.setField(overrideFld, tplRef);\n tx.lock(overrideRef);\n return overrideRef;\n },\n childHashes,\n });\n\n return hash;\n }\n\n processTemplate(data);\n return nodes;\n}\n\nfunction flattenV3Tree(data: CompiledTemplateV3): CacheableNode[] {\n const nodes: CacheableNode[] = [];\n const seen = new Set<string>();\n const sources = data.hashToSource;\n\n function processLib(lib: TemplateLibDataV3): string {\n const hash = hashLibV3(lib);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) =>\n tx.createValue(\n PlTemplateLibV1.type,\n JSON.stringify(\n PlTemplateLibV1.fromV3Data(lib, getSourceCode(lib.name, sources, lib.sourceHash))\n .data,\n ),\n ),\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processSoftware(sw: TemplateSoftwareDataV3): string {\n const hash = hashSoftwareV3(sw);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) => {\n const swData = PlTemplateSoftwareV1.fromV3Data(\n sw,\n getSourceCode(sw.name, sources, sw.sourceHash),\n );\n const ref = tx.createStruct(PlTemplateSoftwareV1.type, swData.data);\n tx.setKValue(ref, PlTemplateSoftwareV1.metaNameKey, JSON.stringify(swData.name));\n tx.lock(ref);\n return ref;\n },\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processWasm(wasm: TemplateWasmDataV3): string {\n const hash = hashWasmV3(wasm);\n if (!seen.has(hash)) {\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx) =>\n tx.createValue(\n PlWasmV1.type,\n JSON.stringify(\n PlWasmV1.fromV3Data(wasm, getSourceCode(wasm.name, sources, wasm.sourceHash)).data,\n ),\n ),\n childHashes: [],\n });\n }\n return hash;\n }\n\n function processTemplate(tpl: TemplateDataV3): string {\n // Process children first (bottom-up)\n const childHashes: string[] = [];\n const children: { fieldName: string; hash: string }[] = [];\n\n for (const [libId, lib] of Object.entries(tpl.libs ?? {})) {\n const h = processLib(lib);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.libPrefix}/${libId}`, hash: h });\n }\n for (const [swId, sw] of Object.entries(tpl.software ?? {})) {\n const h = processSoftware(sw);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.softPrefix}/${swId}`, hash: h });\n }\n for (const [swId, sw] of Object.entries(tpl.assets ?? {})) {\n const h = processSoftware(sw);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.softPrefix}/${swId}`, hash: h });\n }\n for (const [tplId, sub] of Object.entries(tpl.templates ?? {})) {\n const h = processTemplate(sub);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.tplPrefix}/${tplId}`, hash: h });\n }\n for (const [wasmId, wasm] of Object.entries(tpl.wasm ?? {})) {\n const h = processWasm(wasm);\n childHashes.push(h);\n children.push({ fieldName: `${PlTemplateV1.wasmPrefix}/${wasmId}`, hash: h });\n }\n\n // Compose hash from own content + child hash strings (NOT child content).\n // Uses sourceHash directly — it already uniquely identifies the source.\n const h = createHash(\"sha256\")\n .update(PlTemplateV1.type.name)\n .update(PlTemplateV1.type.version)\n .update(tpl.hashOverride ?? \"no-override\")\n .update(tpl.name)\n .update(tpl.version)\n .update(tpl.sourceHash);\n for (const child of children) {\n h.update(\"child:\" + child.fieldName + \":\" + child.hash);\n }\n const hash = h.digest(\"hex\");\n\n if (seen.has(hash)) return hash;\n seen.add(hash);\n nodes.push({\n hash,\n create: (tx, childRefs) => {\n const sourceCode = getSourceCode(tpl.name, sources, tpl.sourceHash);\n const tplRef = tx.createStruct(\n PlTemplateV1.type,\n JSON.stringify(PlTemplateV1.fromV3Data(tpl, sourceCode).data),\n );\n for (const child of children) {\n const fld = field(tplRef, child.fieldName);\n tx.createField(fld, \"Input\");\n tx.setField(fld, notEmpty(childRefs.get(child.hash), `missing child ref ${child.hash}`));\n }\n tx.lock(tplRef);\n\n if (!tpl.hashOverride) return tplRef;\n\n const overrideRef = tx.createStruct(\n PlTemplateOverrideV1.type,\n JSON.stringify(PlTemplateOverrideV1.fromV3Data(tpl)),\n );\n const overrideFld = PlTemplateOverrideV1.tplField(overrideRef);\n tx.createField(overrideFld, \"Service\");\n tx.setField(overrideFld, tplRef);\n tx.lock(overrideRef);\n return overrideRef;\n },\n childHashes,\n });\n\n return hash;\n }\n\n processTemplate(data.template);\n return nodes;\n}\n\n/** Flatten template tree into a topologically ordered list of cacheable nodes (leaves first). */\nexport function flattenTemplateTree(data: TemplateData | CompiledTemplateV3): CacheableNode[] {\n if (data.type === \"pl.tengo-template.v2\") {\n return flattenV2Tree(data);\n } else {\n return flattenV3Tree(data);\n }\n}\n\n// ─── Cache operations ────────────────────────────────────────────────────────\n\n/** In-memory cache for the TemplateCache SignedResourceId per PlClient instance. */\nconst cacheRidMap = new WeakMap<PlClient, SignedResourceId>();\n\n/** Clear the in-memory cacheRid entry (call on errors referencing the cache resource). */\nexport function invalidateTemplateCacheId(pl: PlClient): void {\n cacheRidMap.delete(pl);\n}\n\n/** Find or create the TemplateCache/1 resource on user root. */\nexport async function getOrCreateTemplateCache(pl: PlClient): Promise<SignedResourceId> {\n // Check in-memory cache first (0ms after first call)\n const cached = cacheRidMap.get(pl);\n if (cached !== undefined) return cached;\n\n // Try read-only check\n const existing = await pl.withReadTx(\"templateCache:check\", async (tx) => {\n const fd = await tx.getFieldIfExists(field(pl.clientRoot, TemplateCacheFieldName));\n return fd ? ensureSignedResourceIdNotNull(fd.value) : undefined;\n });\n if (existing) {\n cacheRidMap.set(pl, existing);\n return existing;\n }\n\n const result = await pl.withWriteTx(\"templateCache:init\", async (tx) => {\n // Double-check inside write tx (another instance may have created it)\n const fd = await tx.getFieldIfExists(field(pl.clientRoot, TemplateCacheFieldName));\n if (fd) return ensureSignedResourceIdNotNull(fd.value);\n\n const cache = tx.createStruct(TemplateCacheType);\n tx.createField(field(pl.clientRoot, TemplateCacheFieldName), \"Dynamic\", cache);\n tx.lock(cache);\n await tx.commit();\n return await cache.globalId;\n });\n cacheRidMap.set(pl, result);\n return result;\n}\n\n/** Remove the template cache from user root. */\nexport async function dropTemplateCache(pl: PlClient): Promise<void> {\n await pl.withWriteTx(\"templateCache:drop\", async (tx) => {\n const cacheField = field(pl.clientRoot, TemplateCacheFieldName);\n const fd = await tx.getFieldIfExists(cacheField);\n if (fd) {\n tx.removeField(cacheField);\n await tx.commit();\n }\n });\n invalidateTemplateCacheId(pl);\n}\n\n// ─── GC ──────────────────────────────────────────────────────────────────────\n\n/**\n * Run count-based garbage collection on the template cache.\n * Evicts least-recently-used entries when the cache exceeds maxEntries.\n * Always resets the access counter to 0.\n *\n * @internal exported for testing (maxEntries parameter allows low thresholds in tests)\n * @returns true if entries were evicted\n */\nexport async function runGc(\n pl: PlClient,\n cacheRid: SignedResourceId,\n maxEntries: number = GC_MAX_ENTRIES,\n): Promise<boolean> {\n return await pl.withWriteTx(\"templateCache:gc\", async (tx) => {\n const kvs = await tx.listKeyValuesString(cacheRid);\n const entries: { hash: string; timestamp: number }[] = [];\n for (const { key, value } of kvs) {\n if (!key.startsWith(ACCESS_KEY_PREFIX)) continue;\n entries.push({\n hash: key.slice(ACCESS_KEY_PREFIX.length),\n timestamp: parseInt(value, 10),\n });\n }\n\n // Always reset counter\n tx.setKValue(cacheRid, ACCESS_COUNT_KEY, \"0\");\n\n if (entries.length <= maxEntries) {\n await tx.commit();\n return false;\n }\n\n // Sort oldest first, evict until under limit\n entries.sort((a, b) => a.timestamp - b.timestamp);\n const toEvict = entries.length - maxEntries;\n for (let i = 0; i < toEvict; i++) {\n tx.removeField(field(cacheRid, entries[i].hash));\n tx.deleteKValue(cacheRid, ACCESS_KEY_PREFIX + entries[i].hash);\n }\n\n await tx.commit();\n return true;\n });\n}\n\n// ─── Batched materialization ─────────────────────────────────────────────────\n\n/** Create a batch of cache nodes in the current transaction. */\nfunction createBatchNodes(\n tx: PlTransaction,\n cacheRid: SignedResourceId,\n batch: CacheableNode[],\n resolvedIds: ReadonlyMap<string, SignedResourceId>,\n newRefs: Map<string, ResourceRef>,\n now: string,\n): void {\n for (const node of batch) {\n const childRefs = new Map<string, AnyResourceRef>();\n for (const ch of node.childHashes) {\n const resolved = resolvedIds.get(ch) ?? newRefs.get(ch);\n if (resolved === undefined) {\n throw new Error(`BUG: child ${ch} not resolved`);\n }\n childRefs.set(ch, resolved);\n }\n const ref = node.create(tx, childRefs);\n newRefs.set(node.hash, ref);\n tx.createField(field(cacheRid, node.hash), \"Dynamic\", ref);\n tx.setKValue(cacheRid, ACCESS_KEY_PREFIX + node.hash, now);\n }\n}\n\n/**\n * Materialize a template tree via the cache using \"probe all + batched creation\".\n *\n * Phase 1 (single write tx):\n * - Check existence of ALL hashes in one roundtrip\n * - Happy path: if root cached, update access tracking and return\n * - Otherwise: fetch ResourceIds for all cache hits, create first batch of missing nodes\n *\n * Phase 2..N (one write tx per batch):\n * - Create remaining missing nodes in BATCH_SIZE chunks\n *\n * @returns root SignedResourceId and current access count (for GC decision)\n */\nasync function materialize(\n pl: PlClient,\n cacheRid: SignedResourceId,\n rootHash: string,\n nodes: CacheableNode[],\n stat: TemplateCacheStat,\n): Promise<{ rootId: SignedResourceId; accessCount: number }> {\n const allHashes = nodes.map((n) => n.hash);\n const resolvedIds = new Map<string, SignedResourceId>();\n\n // Phase 1: probe all + first batch\n const phase1 = await pl.withWriteTx(\"templateCache:materialize\", async (tx) => {\n // 1 roundtrip: check all hashes + read access count\n const [exists, countStr] = await Promise.all([\n Promise.all(allHashes.map((h) => tx.fieldExists(field(cacheRid, h)))),\n tx.getKValueStringIfExists(cacheRid, ACCESS_COUNT_KEY),\n ]);\n\n const prevCount = countStr ? parseInt(countStr, 10) : 0;\n const newCount = prevCount + 1;\n const now = Date.now().toString();\n const rootIdx = allHashes.length - 1;\n\n // Happy path: root already cached\n if (exists[rootIdx]) {\n const rootFd = await tx.getField(field(cacheRid, rootHash));\n const rootRid = ensureSignedResourceIdNotNull(rootFd.value);\n tx.setKValue(cacheRid, ACCESS_KEY_PREFIX + rootHash, now);\n tx.setKValue(cacheRid, ACCESS_COUNT_KEY, newCount.toString());\n await tx.commit();\n stat.happyPath = true;\n stat.cacheHits = stat.totalNodes;\n stat.batchCount = 1;\n return { done: true as const, rootId: rootRid, accessCount: newCount };\n }\n\n // Fetch ResourceIds for all cache hits (1 roundtrip)\n const hitIndices: number[] = [];\n for (let i = 0; i < allHashes.length; i++) {\n if (exists[i]) hitIndices.push(i);\n }\n\n if (hitIndices.length > 0) {\n const hitFields = await Promise.all(\n hitIndices.map((i) => tx.getField(field(cacheRid, allHashes[i]))),\n );\n for (let j = 0; j < hitIndices.length; j++) {\n resolvedIds.set(\n allHashes[hitIndices[j]],\n ensureSignedResourceIdNotNull(hitFields[j].value),\n );\n }\n }\n stat.cacheHits = hitIndices.length;\n\n // Missing nodes (topo order preserved from flatten)\n const missing = nodes.filter((n) => !resolvedIds.has(n.hash));\n stat.cacheMisses = missing.length;\n\n // Create first batch of missing nodes\n const firstBatch = missing.slice(0, BATCH_SIZE);\n const newRefs = new Map<string, ResourceRef>();\n createBatchNodes(tx, cacheRid, firstBatch, resolvedIds, newRefs, now);\n\n // Update access tracking for cache hits\n for (const i of hitIndices) {\n tx.setKValue(cacheRid, ACCESS_KEY_PREFIX + allHashes[i], now);\n }\n tx.setKValue(cacheRid, ACCESS_COUNT_KEY, newCount.toString());\n\n await tx.commit();\n\n // Resolve new refs to global IDs (after commit)\n for (const [hash, ref] of newRefs) {\n resolvedIds.set(hash, await toGlobalResourceId(ref));\n }\n\n return {\n done: false as const,\n remaining: missing.slice(BATCH_SIZE),\n accessCount: newCount,\n };\n });\n\n if (phase1.done) {\n return { rootId: phase1.rootId, accessCount: phase1.accessCount };\n }\n\n stat.batchCount = 1;\n\n // Phase 2+: remaining batches\n const { remaining } = phase1;\n for (let i = 0; i < remaining.length; i += BATCH_SIZE) {\n const batch = remaining.slice(i, i + BATCH_SIZE);\n stat.batchCount++;\n\n await pl.withWriteTx(\"templateCache:create\", async (tx) => {\n const newRefs = new Map<string, ResourceRef>();\n const now = Date.now().toString();\n createBatchNodes(tx, cacheRid, batch, resolvedIds, newRefs, now);\n await tx.commit();\n\n for (const [hash, ref] of newRefs) {\n resolvedIds.set(hash, await toGlobalResourceId(ref));\n }\n });\n }\n\n const rootId = resolvedIds.get(rootHash);\n if (!rootId) throw new Error(\"BUG: root hash not resolved after all batches\");\n return { rootId, accessCount: phase1.accessCount };\n}\n\n/**\n * Materialize a template tree via the cache.\n * Manages its own transactions internally — do NOT call inside an existing tx.\n *\n * @returns concrete SignedResourceId of the root template\n */\nexport async function loadTemplateCached(\n pl: PlClient,\n spec: TemplateSpecPrepared,\n options?: { cacheResourceId?: SignedResourceId },\n): Promise<SignedResourceId> {\n const stat = initialStat();\n const t0 = performance.now();\n\n try {\n // Parse to data if needed\n let tplData: TemplateData | CompiledTemplateV3;\n switch (spec.type) {\n case \"explicit\":\n tplData = parseTemplate(spec.content);\n break;\n case \"prepared\":\n tplData = spec.data;\n break;\n case \"cached\":\n return spec.resourceId;\n case \"from-registry\":\n throw new Error(\n \"loadTemplateCached does not support from-registry specs; use loadTemplate instead\",\n );\n default: {\n const _: never = spec;\n throw new Error(`unexpected spec type: ${(_ as any).type}`);\n }\n }\n\n stat.templateFormat = tplData.type;\n\n // Flatten to ordered nodes\n const tFlatten = performance.now();\n const nodes = flattenTemplateTree(tplData);\n stat.flattenMs = performance.now() - tFlatten;\n if (nodes.length === 0) throw new Error(\"template tree produced no nodes\");\n\n stat.totalNodes = nodes.length;\n const rootHash = nodes[nodes.length - 1].hash;\n\n // Resolve or create cache resource\n const tCacheInit = performance.now();\n const cacheRid = options?.cacheResourceId ?? (await getOrCreateTemplateCache(pl));\n stat.cacheInitMs = performance.now() - tCacheInit;\n\n // Retry loop: if a write tx fails (e.g. concurrent GC invalidated a cached resource),\n // restart materialization from scratch.\n const MAX_RETRIES = 3;\n for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {\n try {\n const tMat = performance.now();\n const result = await materialize(pl, cacheRid, rootHash, nodes, stat);\n stat.materializeMs = performance.now() - tMat;\n stat.retries = attempt;\n\n // GC in separate tx if access count exceeded threshold\n if (result.accessCount >= GC_ACCESS_THRESHOLD) {\n await runGc(pl, cacheRid);\n stat.gcTriggered = true;\n }\n\n return result.rootId;\n } catch (e) {\n if (attempt === MAX_RETRIES - 1) throw e;\n // Retry from scratch — previous batch results may reference GC'd resources\n stat.cacheHits = 0;\n stat.cacheMisses = 0;\n }\n }\n\n throw new Error(\"BUG: unreachable\");\n } finally {\n stat.totalMs = performance.now() - t0;\n if (getDebugFlags().logTemplateCacheStat) {\n console.log(`[templateCache] ${JSON.stringify(stat)}`);\n }\n }\n}\n\n// ─── Caller helper ───────────────────────────────────────────────────────────\n\n/**\n * Pre-materialize a block pack's template via cache.\n * Returns a copy of the spec with the template replaced by a cached reference.\n * If the template is already cached, returns the spec unchanged.\n */\nexport async function cacheBlockPackTemplate(\n pl: PlClient,\n spec: BlockPackSpecPrepared,\n options?: { cacheResourceId?: SignedResourceId },\n): Promise<BlockPackSpecPrepared> {\n if (spec.template.type === \"cached\") return spec;\n\n const resourceId = await loadTemplateCached(pl, spec.template, options);\n return {\n ...spec,\n template: { type: \"cached\", resourceId },\n };\n}\n"],"mappings":";;;;;;AAqCA,MAAa,oBAAoB,aAAa,iBAAiB,IAAI;AAEnE,MAAa,yBAAyB;AACtC,MAAM,aAAa;;AAEnB,MAAa,sBAAsB;;AAEnC,MAAa,iBAAiB;;AAE9B,MAAa,mBAAmB;;AAEhC,MAAa,oBAAoB;AAmBjC,SAAS,cAAiC;AACxC,QAAO;EACL,SAAS;EACT,WAAW;EACX,aAAa;EACb,eAAe;EACf,YAAY;EACZ,WAAW;EACX,aAAa;EACb,YAAY;EACZ,WAAW;EACX,aAAa;EACb,SAAS;EACT,gBAAgB;EACjB;;AAiBH,SAAS,cAAc,MAAc,SAAiC,YAA4B;AAChG,QAAO,SACL,QAAQ,aACR,kBAAkB,KAAK,oDAAoD,aAC5E;;;;;;AAUH,SAAS,UAAU,KAA8B;AAC/C,QAAO,WAAW,SAAS,CACxB,OAAO,gBAAgB,KAAK,KAAK,CACjC,OAAO,gBAAgB,KAAK,QAAQ,CACpC,OAAO,IAAI,KAAK,CAChB,OAAO,IAAI,QAAQ,CACnB,OAAO,IAAI,IAAI,CACf,OAAO,MAAM;;AAGlB,SAAS,eAAe,IAAkC;AACxD,QAAO,WAAW,SAAS,CACxB,OAAO,qBAAqB,KAAK,KAAK,CACtC,OAAO,qBAAqB,KAAK,QAAQ,CACzC,OAAO,GAAG,KAAK,CACf,OAAO,GAAG,QAAQ,CAClB,OAAO,GAAG,IAAI,CACd,OAAO,MAAM;;AAKlB,SAAS,UAAU,KAAgC;AACjD,QAAO,WAAW,SAAS,CACxB,OAAO,gBAAgB,KAAK,KAAK,CACjC,OAAO,gBAAgB,KAAK,QAAQ,CACpC,OAAO,IAAI,KAAK,CAChB,OAAO,IAAI,QAAQ,CACnB,OAAO,IAAI,WAAW,CACtB,OAAO,MAAM;;AAGlB,SAAS,eAAe,IAAoC;AAC1D,QAAO,WAAW,SAAS,CACxB,OAAO,qBAAqB,KAAK,KAAK,CACtC,OAAO,qBAAqB,KAAK,QAAQ,CACzC,OAAO,GAAG,KAAK,CACf,OAAO,GAAG,QAAQ,CAClB,OAAO,GAAG,WAAW,CACrB,OAAO,MAAM;;AAGlB,SAAS,WAAW,MAAkC;AACpD,QAAO,WAAW,SAAS,CACxB,OAAO,SAAS,KAAK,KAAK,CAC1B,OAAO,SAAS,KAAK,QAAQ,CAC7B,OAAO,KAAK,KAAK,CACjB,OAAO,KAAK,QAAQ,CACpB,OAAO,KAAK,WAAW,CACvB,OAAO,MAAM;;AAKlB,SAAS,cAAc,MAAqC;CAC1D,MAAM,QAAyB,EAAE;CACjC,MAAM,uBAAO,IAAI,KAAa;CAE9B,SAAS,WAAW,KAA8B;EAChD,MAAM,OAAO,UAAU,IAAI;AAC3B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OACP,GAAG,YACD,gBAAgB,MAChB,KAAK,UAAU,gBAAgB,WAAW,IAAI,CAAC,KAAK,CACrD;IACH,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,gBAAgB,IAAkC;EACzD,MAAM,OAAO,eAAe,GAAG;AAC/B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OAAO;KACd,MAAM,SAAS,qBAAqB,WAAW,GAAG;KAClD,MAAM,MAAM,GAAG,aAAa,qBAAqB,MAAM,OAAO,KAAK;AACnE,QAAG,UAAU,KAAK,qBAAqB,aAAa,KAAK,UAAU,OAAO,KAAK,CAAC;AAChF,QAAG,KAAK,IAAI;AACZ,YAAO;;IAET,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,gBAAgB,KAA2B;EAElD,MAAM,cAAwB,EAAE;EAChC,MAAM,WAAkD,EAAE;AAE1D,OAAK,MAAM,CAAC,OAAO,QAAQ,OAAO,QAAQ,IAAI,QAAQ,EAAE,CAAC,EAAE;GACzD,MAAM,IAAI,WAAW,IAAI;AACzB,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAG,aAAa,UAAU,GAAG;IAAS,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,MAAM,OAAO,OAAO,QAAQ,IAAI,YAAY,EAAE,CAAC,EAAE;GAC3D,MAAM,IAAI,gBAAgB,GAAG;AAC7B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAG,aAAa,WAAW,GAAG;IAAQ,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,MAAM,OAAO,OAAO,QAAQ,IAAI,UAAU,EAAE,CAAC,EAAE;GACzD,MAAM,IAAI,gBAAgB,GAAG;AAC7B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAG,aAAa,WAAW,GAAG;IAAQ,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,OAAO,QAAQ,OAAO,QAAQ,IAAI,aAAa,EAAE,CAAC,EAAE;GAC9D,MAAM,IAAI,gBAAgB,IAAI;AAC9B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAG,aAAa,UAAU,GAAG;IAAS,MAAM;IAAG,CAAC;;EAI7E,MAAM,IAAI,WAAW,SAAS,CAC3B,OAAO,aAAa,KAAK,KAAK,CAC9B,OAAO,aAAa,KAAK,QAAQ,CACjC,OAAO,IAAI,gBAAgB,cAAc,CACzC,OAAO,IAAI,KAAK,CAChB,OAAO,IAAI,QAAQ,CACnB,OAAO,IAAI,IAAI;AAClB,OAAK,MAAM,SAAS,SAClB,GAAE,OAAO,WAAW,MAAM,YAAY,MAAM,MAAM,KAAK;EAEzD,MAAM,OAAO,EAAE,OAAO,MAAM;AAE5B,MAAI,KAAK,IAAI,KAAK,CAAE,QAAO;AAC3B,OAAK,IAAI,KAAK;AACd,QAAM,KAAK;GACT;GACA,SAAS,IAAI,cAAc;IACzB,MAAM,SAAS,GAAG,aAChB,aAAa,MACb,KAAK,UAAU,aAAa,WAAW,IAAI,CAAC,KAAK,CAClD;AACD,SAAK,MAAM,SAAS,UAAU;KAC5B,MAAM,MAAM,MAAM,QAAQ,MAAM,UAAU;AAC1C,QAAG,YAAY,KAAK,QAAQ;AAC5B,QAAG,SAAS,KAAK,SAAS,UAAU,IAAI,MAAM,KAAK,EAAE,qBAAqB,MAAM,OAAO,CAAC;;AAE1F,OAAG,KAAK,OAAO;AAEf,QAAI,CAAC,IAAI,aAAc,QAAO;IAE9B,MAAM,cAAc,GAAG,aACrB,qBAAqB,MACrB,KAAK,UAAU,qBAAqB,WAAW,IAAI,CAAC,CACrD;IACD,MAAM,cAAc,qBAAqB,SAAS,YAAY;AAC9D,OAAG,YAAY,aAAa,UAAU;AACtC,OAAG,SAAS,aAAa,OAAO;AAChC,OAAG,KAAK,YAAY;AACpB,WAAO;;GAET;GACD,CAAC;AAEF,SAAO;;AAGT,iBAAgB,KAAK;AACrB,QAAO;;AAGT,SAAS,cAAc,MAA2C;CAChE,MAAM,QAAyB,EAAE;CACjC,MAAM,uBAAO,IAAI,KAAa;CAC9B,MAAM,UAAU,KAAK;CAErB,SAAS,WAAW,KAAgC;EAClD,MAAM,OAAO,UAAU,IAAI;AAC3B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OACP,GAAG,YACD,gBAAgB,MAChB,KAAK,UACH,gBAAgB,WAAW,KAAK,cAAc,IAAI,MAAM,SAAS,IAAI,WAAW,CAAC,CAC9E,KACJ,CACF;IACH,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,gBAAgB,IAAoC;EAC3D,MAAM,OAAO,eAAe,GAAG;AAC/B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OAAO;KACd,MAAM,SAAS,qBAAqB,WAClC,IACA,cAAc,GAAG,MAAM,SAAS,GAAG,WAAW,CAC/C;KACD,MAAM,MAAM,GAAG,aAAa,qBAAqB,MAAM,OAAO,KAAK;AACnE,QAAG,UAAU,KAAK,qBAAqB,aAAa,KAAK,UAAU,OAAO,KAAK,CAAC;AAChF,QAAG,KAAK,IAAI;AACZ,YAAO;;IAET,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,YAAY,MAAkC;EACrD,MAAM,OAAO,WAAW,KAAK;AAC7B,MAAI,CAAC,KAAK,IAAI,KAAK,EAAE;AACnB,QAAK,IAAI,KAAK;AACd,SAAM,KAAK;IACT;IACA,SAAS,OACP,GAAG,YACD,SAAS,MACT,KAAK,UACH,SAAS,WAAW,MAAM,cAAc,KAAK,MAAM,SAAS,KAAK,WAAW,CAAC,CAAC,KAC/E,CACF;IACH,aAAa,EAAE;IAChB,CAAC;;AAEJ,SAAO;;CAGT,SAAS,gBAAgB,KAA6B;EAEpD,MAAM,cAAwB,EAAE;EAChC,MAAM,WAAkD,EAAE;AAE1D,OAAK,MAAM,CAAC,OAAO,QAAQ,OAAO,QAAQ,IAAI,QAAQ,EAAE,CAAC,EAAE;GACzD,MAAM,IAAI,WAAW,IAAI;AACzB,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAG,aAAa,UAAU,GAAG;IAAS,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,MAAM,OAAO,OAAO,QAAQ,IAAI,YAAY,EAAE,CAAC,EAAE;GAC3D,MAAM,IAAI,gBAAgB,GAAG;AAC7B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAG,aAAa,WAAW,GAAG;IAAQ,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,MAAM,OAAO,OAAO,QAAQ,IAAI,UAAU,EAAE,CAAC,EAAE;GACzD,MAAM,IAAI,gBAAgB,GAAG;AAC7B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAG,aAAa,WAAW,GAAG;IAAQ,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,OAAO,QAAQ,OAAO,QAAQ,IAAI,aAAa,EAAE,CAAC,EAAE;GAC9D,MAAM,IAAI,gBAAgB,IAAI;AAC9B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAG,aAAa,UAAU,GAAG;IAAS,MAAM;IAAG,CAAC;;AAE7E,OAAK,MAAM,CAAC,QAAQ,SAAS,OAAO,QAAQ,IAAI,QAAQ,EAAE,CAAC,EAAE;GAC3D,MAAM,IAAI,YAAY,KAAK;AAC3B,eAAY,KAAK,EAAE;AACnB,YAAS,KAAK;IAAE,WAAW,GAAG,aAAa,WAAW,GAAG;IAAU,MAAM;IAAG,CAAC;;EAK/E,MAAM,IAAI,WAAW,SAAS,CAC3B,OAAO,aAAa,KAAK,KAAK,CAC9B,OAAO,aAAa,KAAK,QAAQ,CACjC,OAAO,IAAI,gBAAgB,cAAc,CACzC,OAAO,IAAI,KAAK,CAChB,OAAO,IAAI,QAAQ,CACnB,OAAO,IAAI,WAAW;AACzB,OAAK,MAAM,SAAS,SAClB,GAAE,OAAO,WAAW,MAAM,YAAY,MAAM,MAAM,KAAK;EAEzD,MAAM,OAAO,EAAE,OAAO,MAAM;AAE5B,MAAI,KAAK,IAAI,KAAK,CAAE,QAAO;AAC3B,OAAK,IAAI,KAAK;AACd,QAAM,KAAK;GACT;GACA,SAAS,IAAI,cAAc;IACzB,MAAM,aAAa,cAAc,IAAI,MAAM,SAAS,IAAI,WAAW;IACnE,MAAM,SAAS,GAAG,aAChB,aAAa,MACb,KAAK,UAAU,aAAa,WAAW,KAAK,WAAW,CAAC,KAAK,CAC9D;AACD,SAAK,MAAM,SAAS,UAAU;KAC5B,MAAM,MAAM,MAAM,QAAQ,MAAM,UAAU;AAC1C,QAAG,YAAY,KAAK,QAAQ;AAC5B,QAAG,SAAS,KAAK,SAAS,UAAU,IAAI,MAAM,KAAK,EAAE,qBAAqB,MAAM,OAAO,CAAC;;AAE1F,OAAG,KAAK,OAAO;AAEf,QAAI,CAAC,IAAI,aAAc,QAAO;IAE9B,MAAM,cAAc,GAAG,aACrB,qBAAqB,MACrB,KAAK,UAAU,qBAAqB,WAAW,IAAI,CAAC,CACrD;IACD,MAAM,cAAc,qBAAqB,SAAS,YAAY;AAC9D,OAAG,YAAY,aAAa,UAAU;AACtC,OAAG,SAAS,aAAa,OAAO;AAChC,OAAG,KAAK,YAAY;AACpB,WAAO;;GAET;GACD,CAAC;AAEF,SAAO;;AAGT,iBAAgB,KAAK,SAAS;AAC9B,QAAO;;;AAIT,SAAgB,oBAAoB,MAA0D;AAC5F,KAAI,KAAK,SAAS,uBAChB,QAAO,cAAc,KAAK;KAE1B,QAAO,cAAc,KAAK;;;AAO9B,MAAM,8BAAc,IAAI,SAAqC;;AAG7D,SAAgB,0BAA0B,IAAoB;AAC5D,aAAY,OAAO,GAAG;;;AAIxB,eAAsB,yBAAyB,IAAyC;CAEtF,MAAM,SAAS,YAAY,IAAI,GAAG;AAClC,KAAI,WAAW,KAAA,EAAW,QAAO;CAGjC,MAAM,WAAW,MAAM,GAAG,WAAW,uBAAuB,OAAO,OAAO;EACxE,MAAM,KAAK,MAAM,GAAG,iBAAiB,MAAM,GAAG,YAAY,uBAAuB,CAAC;AAClF,SAAO,KAAK,8BAA8B,GAAG,MAAM,GAAG,KAAA;GACtD;AACF,KAAI,UAAU;AACZ,cAAY,IAAI,IAAI,SAAS;AAC7B,SAAO;;CAGT,MAAM,SAAS,MAAM,GAAG,YAAY,sBAAsB,OAAO,OAAO;EAEtE,MAAM,KAAK,MAAM,GAAG,iBAAiB,MAAM,GAAG,YAAY,uBAAuB,CAAC;AAClF,MAAI,GAAI,QAAO,8BAA8B,GAAG,MAAM;EAEtD,MAAM,QAAQ,GAAG,aAAa,kBAAkB;AAChD,KAAG,YAAY,MAAM,GAAG,YAAY,uBAAuB,EAAE,WAAW,MAAM;AAC9E,KAAG,KAAK,MAAM;AACd,QAAM,GAAG,QAAQ;AACjB,SAAO,MAAM,MAAM;GACnB;AACF,aAAY,IAAI,IAAI,OAAO;AAC3B,QAAO;;;AAIT,eAAsB,kBAAkB,IAA6B;AACnE,OAAM,GAAG,YAAY,sBAAsB,OAAO,OAAO;EACvD,MAAM,aAAa,MAAM,GAAG,YAAY,uBAAuB;AAE/D,MADW,MAAM,GAAG,iBAAiB,WAAW,EACxC;AACN,MAAG,YAAY,WAAW;AAC1B,SAAM,GAAG,QAAQ;;GAEnB;AACF,2BAA0B,GAAG;;;;;;;;;;AAa/B,eAAsB,MACpB,IACA,UACA,aAAqB,gBACH;AAClB,QAAO,MAAM,GAAG,YAAY,oBAAoB,OAAO,OAAO;EAC5D,MAAM,MAAM,MAAM,GAAG,oBAAoB,SAAS;EAClD,MAAM,UAAiD,EAAE;AACzD,OAAK,MAAM,EAAE,KAAK,WAAW,KAAK;AAChC,OAAI,CAAC,IAAI,WAAA,UAA6B,CAAE;AACxC,WAAQ,KAAK;IACX,MAAM,IAAI,MAAM,EAAyB;IACzC,WAAW,SAAS,OAAO,GAAG;IAC/B,CAAC;;AAIJ,KAAG,UAAU,UAAU,kBAAkB,IAAI;AAE7C,MAAI,QAAQ,UAAU,YAAY;AAChC,SAAM,GAAG,QAAQ;AACjB,UAAO;;AAIT,UAAQ,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,UAAU;EACjD,MAAM,UAAU,QAAQ,SAAS;AACjC,OAAK,IAAI,IAAI,GAAG,IAAI,SAAS,KAAK;AAChC,MAAG,YAAY,MAAM,UAAU,QAAQ,GAAG,KAAK,CAAC;AAChD,MAAG,aAAa,UAAU,oBAAoB,QAAQ,GAAG,KAAK;;AAGhE,QAAM,GAAG,QAAQ;AACjB,SAAO;GACP;;;AAMJ,SAAS,iBACP,IACA,UACA,OACA,aACA,SACA,KACM;AACN,MAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,4BAAY,IAAI,KAA6B;AACnD,OAAK,MAAM,MAAM,KAAK,aAAa;GACjC,MAAM,WAAW,YAAY,IAAI,GAAG,IAAI,QAAQ,IAAI,GAAG;AACvD,OAAI,aAAa,KAAA,EACf,OAAM,IAAI,MAAM,cAAc,GAAG,eAAe;AAElD,aAAU,IAAI,IAAI,SAAS;;EAE7B,MAAM,MAAM,KAAK,OAAO,IAAI,UAAU;AACtC,UAAQ,IAAI,KAAK,MAAM,IAAI;AAC3B,KAAG,YAAY,MAAM,UAAU,KAAK,KAAK,EAAE,WAAW,IAAI;AAC1D,KAAG,UAAU,UAAU,oBAAoB,KAAK,MAAM,IAAI;;;;;;;;;;;;;;;;AAiB9D,eAAe,YACb,IACA,UACA,UACA,OACA,MAC4D;CAC5D,MAAM,YAAY,MAAM,KAAK,MAAM,EAAE,KAAK;CAC1C,MAAM,8BAAc,IAAI,KAA+B;CAGvD,MAAM,SAAS,MAAM,GAAG,YAAY,6BAA6B,OAAO,OAAO;EAE7E,MAAM,CAAC,QAAQ,YAAY,MAAM,QAAQ,IAAI,CAC3C,QAAQ,IAAI,UAAU,KAAK,MAAM,GAAG,YAAY,MAAM,UAAU,EAAE,CAAC,CAAC,CAAC,EACrE,GAAG,wBAAwB,UAAU,iBAAiB,CACvD,CAAC;EAGF,MAAM,YADY,WAAW,SAAS,UAAU,GAAG,GAAG,KACzB;EAC7B,MAAM,MAAM,KAAK,KAAK,CAAC,UAAU;AAIjC,MAAI,OAHY,UAAU,SAAS,IAGd;GAEnB,MAAM,UAAU,+BADD,MAAM,GAAG,SAAS,MAAM,UAAU,SAAS,CAAC,EACN,MAAM;AAC3D,MAAG,UAAU,UAAU,oBAAoB,UAAU,IAAI;AACzD,MAAG,UAAU,UAAU,kBAAkB,SAAS,UAAU,CAAC;AAC7D,SAAM,GAAG,QAAQ;AACjB,QAAK,YAAY;AACjB,QAAK,YAAY,KAAK;AACtB,QAAK,aAAa;AAClB,UAAO;IAAE,MAAM;IAAe,QAAQ;IAAS,aAAa;IAAU;;EAIxE,MAAM,aAAuB,EAAE;AAC/B,OAAK,IAAI,IAAI,GAAG,IAAI,UAAU,QAAQ,IACpC,KAAI,OAAO,GAAI,YAAW,KAAK,EAAE;AAGnC,MAAI,WAAW,SAAS,GAAG;GACzB,MAAM,YAAY,MAAM,QAAQ,IAC9B,WAAW,KAAK,MAAM,GAAG,SAAS,MAAM,UAAU,UAAU,GAAG,CAAC,CAAC,CAClE;AACD,QAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,IACrC,aAAY,IACV,UAAU,WAAW,KACrB,8BAA8B,UAAU,GAAG,MAAM,CAClD;;AAGL,OAAK,YAAY,WAAW;EAG5B,MAAM,UAAU,MAAM,QAAQ,MAAM,CAAC,YAAY,IAAI,EAAE,KAAK,CAAC;AAC7D,OAAK,cAAc,QAAQ;EAG3B,MAAM,aAAa,QAAQ,MAAM,GAAG,WAAW;EAC/C,MAAM,0BAAU,IAAI,KAA0B;AAC9C,mBAAiB,IAAI,UAAU,YAAY,aAAa,SAAS,IAAI;AAGrE,OAAK,MAAM,KAAK,WACd,IAAG,UAAU,UAAU,oBAAoB,UAAU,IAAI,IAAI;AAE/D,KAAG,UAAU,UAAU,kBAAkB,SAAS,UAAU,CAAC;AAE7D,QAAM,GAAG,QAAQ;AAGjB,OAAK,MAAM,CAAC,MAAM,QAAQ,QACxB,aAAY,IAAI,MAAM,MAAM,mBAAmB,IAAI,CAAC;AAGtD,SAAO;GACL,MAAM;GACN,WAAW,QAAQ,MAAM,WAAW;GACpC,aAAa;GACd;GACD;AAEF,KAAI,OAAO,KACT,QAAO;EAAE,QAAQ,OAAO;EAAQ,aAAa,OAAO;EAAa;AAGnE,MAAK,aAAa;CAGlB,MAAM,EAAE,cAAc;AACtB,MAAK,IAAI,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK,YAAY;EACrD,MAAM,QAAQ,UAAU,MAAM,GAAG,IAAI,WAAW;AAChD,OAAK;AAEL,QAAM,GAAG,YAAY,wBAAwB,OAAO,OAAO;GACzD,MAAM,0BAAU,IAAI,KAA0B;AAE9C,oBAAiB,IAAI,UAAU,OAAO,aAAa,SADvC,KAAK,KAAK,CAAC,UAAU,CAC+B;AAChE,SAAM,GAAG,QAAQ;AAEjB,QAAK,MAAM,CAAC,MAAM,QAAQ,QACxB,aAAY,IAAI,MAAM,MAAM,mBAAmB,IAAI,CAAC;IAEtD;;CAGJ,MAAM,SAAS,YAAY,IAAI,SAAS;AACxC,KAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,gDAAgD;AAC7E,QAAO;EAAE;EAAQ,aAAa,OAAO;EAAa;;;;;;;;AASpD,eAAsB,mBACpB,IACA,MACA,SAC2B;CAC3B,MAAM,OAAO,aAAa;CAC1B,MAAM,KAAK,YAAY,KAAK;AAE5B,KAAI;EAEF,IAAI;AACJ,UAAQ,KAAK,MAAb;GACE,KAAK;AACH,cAAU,cAAc,KAAK,QAAQ;AACrC;GACF,KAAK;AACH,cAAU,KAAK;AACf;GACF,KAAK,SACH,QAAO,KAAK;GACd,KAAK,gBACH,OAAM,IAAI,MACR,oFACD;GACH,QAEE,OAAM,IAAI,MAAM,yBADC,KACmC,OAAO;;AAI/D,OAAK,iBAAiB,QAAQ;EAG9B,MAAM,WAAW,YAAY,KAAK;EAClC,MAAM,QAAQ,oBAAoB,QAAQ;AAC1C,OAAK,YAAY,YAAY,KAAK,GAAG;AACrC,MAAI,MAAM,WAAW,EAAG,OAAM,IAAI,MAAM,kCAAkC;AAE1E,OAAK,aAAa,MAAM;EACxB,MAAM,WAAW,MAAM,MAAM,SAAS,GAAG;EAGzC,MAAM,aAAa,YAAY,KAAK;EACpC,MAAM,WAAW,SAAS,mBAAoB,MAAM,yBAAyB,GAAG;AAChF,OAAK,cAAc,YAAY,KAAK,GAAG;EAIvC,MAAM,cAAc;AACpB,OAAK,IAAI,UAAU,GAAG,UAAU,aAAa,UAC3C,KAAI;GACF,MAAM,OAAO,YAAY,KAAK;GAC9B,MAAM,SAAS,MAAM,YAAY,IAAI,UAAU,UAAU,OAAO,KAAK;AACrE,QAAK,gBAAgB,YAAY,KAAK,GAAG;AACzC,QAAK,UAAU;AAGf,OAAI,OAAO,eAAA,IAAoC;AAC7C,UAAM,MAAM,IAAI,SAAS;AACzB,SAAK,cAAc;;AAGrB,UAAO,OAAO;WACP,GAAG;AACV,OAAI,YAAY,cAAc,EAAG,OAAM;AAEvC,QAAK,YAAY;AACjB,QAAK,cAAc;;AAIvB,QAAM,IAAI,MAAM,mBAAmB;WAC3B;AACR,OAAK,UAAU,YAAY,KAAK,GAAG;AACnC,MAAI,eAAe,CAAC,qBAClB,SAAQ,IAAI,mBAAmB,KAAK,UAAU,KAAK,GAAG;;;;;;;;AAY5D,eAAsB,uBACpB,IACA,MACA,SACgC;AAChC,KAAI,KAAK,SAAS,SAAS,SAAU,QAAO;CAE5C,MAAM,aAAa,MAAM,mBAAmB,IAAI,KAAK,UAAU,QAAQ;AACvE,QAAO;EACL,GAAG;EACH,UAAU;GAAE,MAAM;GAAU;GAAY;EACzC"}
|
package/dist/pool/driver.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver.cjs","names":["RefCountPoolBase","PFrameDriverError","PFrameInternal","Readable","HttpHelpers","AbstractPFrameDriverOpsDefaults","path","parseDataInfoResource","traverseParquetChunkResource","makeLocalBlobRef","AbstractPFrameDriver"],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n mapDataInfo,\n isDataInfo,\n ensureError,\n PFrameDriverError,\n isAbortError,\n type LocalBlobHandleAndSize,\n type RemoteBlobHandleAndSize,\n type RemoteBlobHandle,\n type ContentHandler,\n type PColumnSpec,\n type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n makeJsonDataInfo,\n AbstractPFrameDriver,\n AbstractPFrameDriverOpsDefaults,\n type AbstractInternalPFrameDriver,\n type AbstractPFrameDriverOps,\n type LocalBlobProvider,\n type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n BlobResourceRef,\n makeLocalBlobRef,\n parseDataInfoResource,\n traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n implements LocalBlobProvider<BlobResourceRef>\n{\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): LocalBlob {\n return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n return resource;\n }\n\n public makeDataSource(\n signal: AbortSignal,\n ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n return {\n preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n try {\n await Promise.all(\n blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n );\n } catch (err: unknown) {\n if (!isAbortError(err)) throw err;\n }\n },\n resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n const computable = this.getByKey(blobId);\n const blob = await computable.awaitStableValue(signal);\n return await this.blobDriver.getContentDirect(blob.handle, { signal });\n },\n };\n }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n BlobResourceRef,\n PFrameInternal.PFrameBlobId,\n RemoteBlob\n> {\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): RemoteBlob {\n if (params.onDemandSnapshot === undefined) {\n throw new PFrameDriverError(\n `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n );\n }\n return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n return resource;\n }\n\n public async withContent<T>(\n handle: RemoteBlobHandle,\n options: {\n range: PFrameInternal.FileRange;\n signal: AbortSignal;\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n return await this.blobDriver.withContent(handle, {\n range: {\n from: options.range.start,\n to: options.range.end + 1,\n },\n signal: options.signal,\n handler: options.handler,\n });\n }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n private readonly remoteBlobProvider: RemoteBlobPool;\n\n constructor(options: BlobStoreOptions) {\n super(options);\n this.remoteBlobProvider = options.remoteBlobProvider;\n }\n\n public override async request(\n filename: PFrameInternal.ParquetFileName,\n params: {\n method: PFrameInternal.HttpMethod;\n range?: PFrameInternal.HttpRange;\n signal: AbortSignal;\n callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n },\n ): Promise<void> {\n const blobId = filename.slice(\n 0,\n -PFrameInternal.ParquetExtension.length,\n ) as PFrameInternal.PFrameBlobId;\n const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n try {\n await params.callback(response);\n } catch (error: unknown) {\n this.logger(\n \"warn\",\n `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n );\n }\n };\n\n try {\n const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n if (!computable) return await respond({ type: \"NotFound\" });\n\n let blob: RemoteBlobHandleAndSize;\n try {\n blob = await computable.getValue();\n } catch (error: unknown) {\n this.logger(\n \"error\",\n `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n );\n return await respond({ type: \"InternalError\" });\n }\n params.signal.throwIfAborted();\n\n const translatedRange = this.translate(blob.size, params.range);\n if (!translatedRange) {\n return await respond({\n type: \"RangeNotSatisfiable\",\n size: blob.size,\n });\n }\n\n if (params.method === \"HEAD\") {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n });\n }\n\n this.logger(\n \"info\",\n `PFrames blob store requesting content for ${blobId}, ` +\n `range [${translatedRange.start}..=${translatedRange.end}]`,\n );\n return await this.remoteBlobProvider.withContent(blob.handle, {\n range: translatedRange,\n signal: params.signal,\n handler: async (data) => {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n data: Readable.fromWeb(data),\n });\n },\n });\n } catch (error: unknown) {\n if (!isAbortError(error)) {\n if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n } else {\n this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n }\n }\n return await respond({ type: \"InternalError\" });\n }\n }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n constructor(\n private readonly pool: RemoteBlobPool,\n private readonly server: PFrameInternal.HttpServer,\n private readonly cache: PFrameInternal.CachingObjectStore,\n ) {}\n\n public static async init(\n blobDriver: DownloadDriver,\n logger: PFrameInternal.Logger,\n serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n cacheConfig: PFrameInternal.CacheConfig,\n ): Promise<RemoteBlobProviderImpl> {\n const pool = new RemoteBlobPool(blobDriver, logger);\n const store = new BlobStore({ remoteBlobProvider: pool, logger });\n const cachingStore = await HttpHelpers.createCachingObjectStore({\n upstream: store,\n config: cacheConfig,\n logger,\n });\n\n const handler = HttpHelpers.createRequestHandler({ store: cachingStore });\n const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n return new RemoteBlobProviderImpl(pool, server, cachingStore);\n }\n\n public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n return this.pool.acquire(params);\n }\n\n public httpServerInfo(): PFrameInternal.HttpServerInfo {\n return this.server.info;\n }\n\n public async getCacheMetrics(): Promise<PFrameInternal.CacheMetrics | null> {\n return this.cache.getMetrics();\n }\n\n public async resetCache(): Promise<void> {\n await this.cache.reset();\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.server.stop();\n await this.cache[Symbol.asyncDispose]();\n await this.pool[Symbol.asyncDispose]();\n }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n /** Port to run parquet HTTP server on. */\n parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n ...AbstractPFrameDriverOpsDefaults,\n parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\n/**\n * Tuning for the persistent parquet blob cache, minus its filesystem path (which\n * is supplied separately - see {@link createPFrameDriver}'s `cachePath`). Carries\n * only the size/eviction knobs of {@link PFrameInternal.CacheConfig}.\n */\nexport type ParquetCacheOps = Omit<PFrameInternal.CacheConfig, \"cachePath\">;\n\nexport async function createPFrameDriver(params: {\n blobDriver: DownloadDriver;\n logger: MiLogger;\n spillPath: string;\n cachePath: string;\n options: PFrameDriverOps;\n cacheOps: ParquetCacheOps;\n}): Promise<InternalPFrameDriver> {\n const resolvedSpillPath = path.resolve(params.spillPath);\n await emptyDir(resolvedSpillPath);\n\n const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n const remoteBlobProvider = await RemoteBlobProviderImpl.init(\n params.blobDriver,\n logger,\n { port: params.options.parquetServerPort },\n { cachePath: path.resolve(params.cachePath), ...params.cacheOps },\n );\n\n const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n if (isPlTreeNodeAccessor(data)) {\n return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n }\n\n return isDataInfo(data)\n ? data.type === \"ParquetPartitioned\"\n ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n : makeJsonDataInfo(spec, data);\n };\n\n return new AbstractPFrameDriver({\n logger,\n localBlobProvider,\n remoteBlobProvider,\n spillPath: resolvedSpillPath,\n options: params.options,\n resolveDataInfo,\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,cAAA,GAAA,0BAAA,uBAAmC,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACUA,wBAAAA,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,qBAAAA,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,EAAA,GAAA,qBAAA,cAAc,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,iBAAiB,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEzE;;;AAKL,IAAM,iBAAN,cAA6BD,wBAAAA,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAIC,qBAAAA,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIA,qBAAAA,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwBC,sCAAAA,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAACA,sCAAAA,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,uEAAA,GAAA,qBAAA,aAAkF,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,2DAAA,GAAA,qBAAA,aAAsE,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAMC,YAAAA,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,EAAA,GAAA,qBAAA,cAAc,MAAM,CACtB,MAAA,GAAA,2BAAA,2BAA8B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,yCAAA,GAAA,qBAAA,aAAoD,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,wCAAA,GAAA,qBAAA,aAAmD,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA,OACA;AAHiB,OAAA,OAAA;AACA,OAAA,SAAA;AACA,OAAA,QAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACA,aACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EACjE,MAAM,eAAe,MAAMC,gCAAAA,YAAY,yBAAyB;GAC9D,UAAU;GACV,QAAQ;GACR;GACD,CAAC;EAEF,MAAM,UAAUA,gCAAAA,YAAY,qBAAqB,EAAE,OAAO,cAAc,CAAC;EACzE,MAAM,SAAS,MAAMA,gCAAAA,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,QAAQ,aAAa;;CAG/D,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,MAAa,kBAA+D;AAC1E,SAAO,KAAK,MAAM,YAAY;;CAGhC,MAAa,aAA4B;AACvC,QAAM,KAAK,MAAM,OAAO;;CAG1B,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,MAAM,OAAO,eAAe;AACvC,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAGC,0BAAAA;CACH,mBAAmB;CACpB;AASD,eAAsB,mBAAmB,QAOP;CAChC,MAAM,oBAAoBC,UAAAA,QAAK,QAAQ,OAAO,UAAU;AACxD,QAAA,GAAA,2BAAA,UAAe,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KACtD,OAAO,YACP,QACA,EAAE,MAAM,OAAO,QAAQ,mBAAmB,EAC1C;EAAE,WAAWA,UAAAA,QAAK,QAAQ,OAAO,UAAU;EAAE,GAAG,OAAO;EAAU,CAClE;CAED,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,OAAA,GAAA,wBAAA,sBAAyB,KAAK,CAC5B,QAAOC,aAAAA,sBAAsB,KAAK,KAAA,GAAA,0BAAA,kBAAqB,MAAM,EAAE,CAAC;AAGlE,UAAA,GAAA,qBAAA,YAAkB,KAAK,GACnB,KAAK,SAAS,wBAAA,GAAA,qBAAA,aACA,OAAO,MAAMC,aAAAA,6BAA6B,EAAE,CAAC,IAAA,GAAA,qBAAA,aAC7C,OAAO,MAAMC,aAAAA,iBAAiB,EAAE,CAAC,IAAA,GAAA,0BAAA,kBAC9B,MAAM,KAAK;;AAGlC,QAAO,IAAIC,0BAAAA,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
|
|
1
|
+
{"version":3,"file":"driver.cjs","names":["RefCountPoolBase","PFrameDriverError","PFrameInternal","Readable","HttpHelpers","AbstractPFrameDriverOpsDefaults","path","parseDataInfoResource","traverseParquetChunkResource","makeLocalBlobRef","AbstractPFrameDriver"],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n mapDataInfo,\n isDataInfo,\n ensureError,\n PFrameDriverError,\n isAbortError,\n type LocalBlobHandleAndSize,\n type RemoteBlobHandleAndSize,\n type RemoteBlobHandle,\n type ContentHandler,\n type PColumnSpec,\n type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n makeJsonDataInfo,\n AbstractPFrameDriver,\n AbstractPFrameDriverOpsDefaults,\n type AbstractInternalPFrameDriver,\n type AbstractPFrameDriverOps,\n type LocalBlobProvider,\n type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n BlobResourceRef,\n makeLocalBlobRef,\n parseDataInfoResource,\n traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n implements LocalBlobProvider<BlobResourceRef>\n{\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): LocalBlob {\n return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n return resource;\n }\n\n public makeDataSource(\n signal: AbortSignal,\n ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n return {\n preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n try {\n await Promise.all(\n blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n );\n } catch (err: unknown) {\n if (!isAbortError(err)) throw err;\n }\n },\n resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n const computable = this.getByKey(blobId);\n const blob = await computable.awaitStableValue(signal);\n return await this.blobDriver.getContentDirect(blob.handle, { signal });\n },\n };\n }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n BlobResourceRef,\n PFrameInternal.PFrameBlobId,\n RemoteBlob\n> {\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): RemoteBlob {\n if (params.onDemandSnapshot === undefined) {\n throw new PFrameDriverError(\n `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n );\n }\n return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n return resource;\n }\n\n public async withContent<T>(\n handle: RemoteBlobHandle,\n options: {\n range: PFrameInternal.FileRange;\n signal: AbortSignal;\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n return await this.blobDriver.withContent(handle, {\n range: {\n from: options.range.start,\n to: options.range.end + 1,\n },\n signal: options.signal,\n handler: options.handler,\n });\n }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n private readonly remoteBlobProvider: RemoteBlobPool;\n\n constructor(options: BlobStoreOptions) {\n super(options);\n this.remoteBlobProvider = options.remoteBlobProvider;\n }\n\n public override async request(\n filename: PFrameInternal.ParquetFileName,\n params: {\n method: PFrameInternal.HttpMethod;\n range?: PFrameInternal.HttpRange;\n signal: AbortSignal;\n callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n },\n ): Promise<void> {\n const blobId = filename.slice(\n 0,\n -PFrameInternal.ParquetExtension.length,\n ) as PFrameInternal.PFrameBlobId;\n const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n try {\n await params.callback(response);\n } catch (error: unknown) {\n this.logger(\n \"warn\",\n `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n );\n }\n };\n\n try {\n const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n if (!computable) return await respond({ type: \"NotFound\" });\n\n let blob: RemoteBlobHandleAndSize;\n try {\n blob = await computable.getValue();\n } catch (error: unknown) {\n this.logger(\n \"error\",\n `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n );\n return await respond({ type: \"InternalError\" });\n }\n params.signal.throwIfAborted();\n\n const translatedRange = this.translate(blob.size, params.range);\n if (!translatedRange) {\n return await respond({\n type: \"RangeNotSatisfiable\",\n size: blob.size,\n });\n }\n\n if (params.method === \"HEAD\") {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n });\n }\n\n this.logger(\n \"info\",\n `PFrames blob store requesting content for ${blobId}, ` +\n `range [${translatedRange.start}..=${translatedRange.end}]`,\n );\n return await this.remoteBlobProvider.withContent(blob.handle, {\n range: translatedRange,\n signal: params.signal,\n handler: async (data) => {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n data: Readable.fromWeb(data),\n });\n },\n });\n } catch (error: unknown) {\n if (!isAbortError(error)) {\n if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n } else {\n this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n }\n }\n return await respond({ type: \"InternalError\" });\n }\n }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n constructor(\n private readonly pool: RemoteBlobPool,\n private readonly server: PFrameInternal.HttpServer,\n private readonly cache: PFrameInternal.CachingObjectStore,\n ) {}\n\n public static async init(\n blobDriver: DownloadDriver,\n logger: PFrameInternal.Logger,\n serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n cacheConfig: PFrameInternal.CacheConfig,\n ): Promise<RemoteBlobProviderImpl> {\n const pool = new RemoteBlobPool(blobDriver, logger);\n const store = new BlobStore({ remoteBlobProvider: pool, logger });\n const cachingStore = (await HttpHelpers.createCachingObjectStore({\n upstream: store,\n config: cacheConfig,\n logger,\n })) as PFrameInternal.CachingObjectStore; // TODO: remove cast after next PFrames release\n\n const handler = HttpHelpers.createRequestHandler({ store: cachingStore });\n const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n return new RemoteBlobProviderImpl(pool, server, cachingStore);\n }\n\n public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n return this.pool.acquire(params);\n }\n\n public httpServerInfo(): PFrameInternal.HttpServerInfo {\n return this.server.info;\n }\n\n public async getCacheMetrics(): Promise<PFrameInternal.CacheMetrics | null> {\n return this.cache.getMetrics();\n }\n\n public async resetCache(): Promise<void> {\n await this.cache.reset();\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.server.stop();\n await this.cache[Symbol.asyncDispose]();\n await this.pool[Symbol.asyncDispose]();\n }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n /** Port to run parquet HTTP server on. */\n parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n ...AbstractPFrameDriverOpsDefaults,\n parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\n/**\n * Tuning for the persistent parquet blob cache, minus its filesystem path (which\n * is supplied separately - see {@link createPFrameDriver}'s `cachePath`). Carries\n * only the size/eviction knobs of {@link PFrameInternal.CacheConfig}.\n */\nexport type ParquetCacheOps = Omit<PFrameInternal.CacheConfig, \"cachePath\">;\n\nexport async function createPFrameDriver(params: {\n blobDriver: DownloadDriver;\n logger: MiLogger;\n spillPath: string;\n cachePath: string;\n options: PFrameDriverOps;\n cacheOps: ParquetCacheOps;\n}): Promise<InternalPFrameDriver> {\n const resolvedSpillPath = path.resolve(params.spillPath);\n await emptyDir(resolvedSpillPath);\n\n const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n const remoteBlobProvider = await RemoteBlobProviderImpl.init(\n params.blobDriver,\n logger,\n { port: params.options.parquetServerPort },\n { cachePath: path.resolve(params.cachePath), ...params.cacheOps },\n );\n\n const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n if (isPlTreeNodeAccessor(data)) {\n return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n }\n\n return isDataInfo(data)\n ? data.type === \"ParquetPartitioned\"\n ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n : makeJsonDataInfo(spec, data);\n };\n\n return new AbstractPFrameDriver({\n logger,\n localBlobProvider,\n remoteBlobProvider,\n spillPath: resolvedSpillPath,\n options: params.options,\n resolveDataInfo,\n });\n}\n"],"mappings":";;;;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,cAAA,GAAA,0BAAA,uBAAmC,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACUA,wBAAAA,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIC,qBAAAA,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,EAAA,GAAA,qBAAA,cAAc,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,iBAAiB,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEzE;;;AAKL,IAAM,iBAAN,cAA6BD,wBAAAA,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAIC,qBAAAA,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAIA,qBAAAA,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwBC,sCAAAA,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAACA,sCAAAA,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,uEAAA,GAAA,qBAAA,aAAkF,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,2DAAA,GAAA,qBAAA,aAAsE,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAMC,YAAAA,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,EAAA,GAAA,qBAAA,cAAc,MAAM,CACtB,MAAA,GAAA,2BAAA,2BAA8B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,yCAAA,GAAA,qBAAA,aAAoD,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,wCAAA,GAAA,qBAAA,aAAmD,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA,OACA;AAHiB,OAAA,OAAA;AACA,OAAA,SAAA;AACA,OAAA,QAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACA,aACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EACjE,MAAM,eAAgB,MAAMC,gCAAAA,YAAY,yBAAyB;GAC/D,UAAU;GACV,QAAQ;GACR;GACD,CAAC;EAEF,MAAM,UAAUA,gCAAAA,YAAY,qBAAqB,EAAE,OAAO,cAAc,CAAC;EACzE,MAAM,SAAS,MAAMA,gCAAAA,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,QAAQ,aAAa;;CAG/D,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,MAAa,kBAA+D;AAC1E,SAAO,KAAK,MAAM,YAAY;;CAGhC,MAAa,aAA4B;AACvC,QAAM,KAAK,MAAM,OAAO;;CAG1B,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,MAAM,OAAO,eAAe;AACvC,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAGC,0BAAAA;CACH,mBAAmB;CACpB;AASD,eAAsB,mBAAmB,QAOP;CAChC,MAAM,oBAAoBC,UAAAA,QAAK,QAAQ,OAAO,UAAU;AACxD,QAAA,GAAA,2BAAA,UAAe,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KACtD,OAAO,YACP,QACA,EAAE,MAAM,OAAO,QAAQ,mBAAmB,EAC1C;EAAE,WAAWA,UAAAA,QAAK,QAAQ,OAAO,UAAU;EAAE,GAAG,OAAO;EAAU,CAClE;CAED,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,OAAA,GAAA,wBAAA,sBAAyB,KAAK,CAC5B,QAAOC,aAAAA,sBAAsB,KAAK,KAAA,GAAA,0BAAA,kBAAqB,MAAM,EAAE,CAAC;AAGlE,UAAA,GAAA,qBAAA,YAAkB,KAAK,GACnB,KAAK,SAAS,wBAAA,GAAA,qBAAA,aACA,OAAO,MAAMC,aAAAA,6BAA6B,EAAE,CAAC,IAAA,GAAA,qBAAA,aAC7C,OAAO,MAAMC,aAAAA,iBAAiB,EAAE,CAAC,IAAA,GAAA,0BAAA,kBAC9B,MAAM,KAAK;;AAGlC,QAAO,IAAIC,0BAAAA,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
|
package/dist/pool/driver.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver.js","names":[],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n mapDataInfo,\n isDataInfo,\n ensureError,\n PFrameDriverError,\n isAbortError,\n type LocalBlobHandleAndSize,\n type RemoteBlobHandleAndSize,\n type RemoteBlobHandle,\n type ContentHandler,\n type PColumnSpec,\n type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n makeJsonDataInfo,\n AbstractPFrameDriver,\n AbstractPFrameDriverOpsDefaults,\n type AbstractInternalPFrameDriver,\n type AbstractPFrameDriverOps,\n type LocalBlobProvider,\n type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n BlobResourceRef,\n makeLocalBlobRef,\n parseDataInfoResource,\n traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n implements LocalBlobProvider<BlobResourceRef>\n{\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): LocalBlob {\n return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n return resource;\n }\n\n public makeDataSource(\n signal: AbortSignal,\n ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n return {\n preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n try {\n await Promise.all(\n blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n );\n } catch (err: unknown) {\n if (!isAbortError(err)) throw err;\n }\n },\n resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n const computable = this.getByKey(blobId);\n const blob = await computable.awaitStableValue(signal);\n return await this.blobDriver.getContentDirect(blob.handle, { signal });\n },\n };\n }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n BlobResourceRef,\n PFrameInternal.PFrameBlobId,\n RemoteBlob\n> {\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): RemoteBlob {\n if (params.onDemandSnapshot === undefined) {\n throw new PFrameDriverError(\n `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n );\n }\n return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n return resource;\n }\n\n public async withContent<T>(\n handle: RemoteBlobHandle,\n options: {\n range: PFrameInternal.FileRange;\n signal: AbortSignal;\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n return await this.blobDriver.withContent(handle, {\n range: {\n from: options.range.start,\n to: options.range.end + 1,\n },\n signal: options.signal,\n handler: options.handler,\n });\n }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n private readonly remoteBlobProvider: RemoteBlobPool;\n\n constructor(options: BlobStoreOptions) {\n super(options);\n this.remoteBlobProvider = options.remoteBlobProvider;\n }\n\n public override async request(\n filename: PFrameInternal.ParquetFileName,\n params: {\n method: PFrameInternal.HttpMethod;\n range?: PFrameInternal.HttpRange;\n signal: AbortSignal;\n callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n },\n ): Promise<void> {\n const blobId = filename.slice(\n 0,\n -PFrameInternal.ParquetExtension.length,\n ) as PFrameInternal.PFrameBlobId;\n const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n try {\n await params.callback(response);\n } catch (error: unknown) {\n this.logger(\n \"warn\",\n `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n );\n }\n };\n\n try {\n const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n if (!computable) return await respond({ type: \"NotFound\" });\n\n let blob: RemoteBlobHandleAndSize;\n try {\n blob = await computable.getValue();\n } catch (error: unknown) {\n this.logger(\n \"error\",\n `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n );\n return await respond({ type: \"InternalError\" });\n }\n params.signal.throwIfAborted();\n\n const translatedRange = this.translate(blob.size, params.range);\n if (!translatedRange) {\n return await respond({\n type: \"RangeNotSatisfiable\",\n size: blob.size,\n });\n }\n\n if (params.method === \"HEAD\") {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n });\n }\n\n this.logger(\n \"info\",\n `PFrames blob store requesting content for ${blobId}, ` +\n `range [${translatedRange.start}..=${translatedRange.end}]`,\n );\n return await this.remoteBlobProvider.withContent(blob.handle, {\n range: translatedRange,\n signal: params.signal,\n handler: async (data) => {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n data: Readable.fromWeb(data),\n });\n },\n });\n } catch (error: unknown) {\n if (!isAbortError(error)) {\n if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n } else {\n this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n }\n }\n return await respond({ type: \"InternalError\" });\n }\n }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n constructor(\n private readonly pool: RemoteBlobPool,\n private readonly server: PFrameInternal.HttpServer,\n private readonly cache: PFrameInternal.CachingObjectStore,\n ) {}\n\n public static async init(\n blobDriver: DownloadDriver,\n logger: PFrameInternal.Logger,\n serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n cacheConfig: PFrameInternal.CacheConfig,\n ): Promise<RemoteBlobProviderImpl> {\n const pool = new RemoteBlobPool(blobDriver, logger);\n const store = new BlobStore({ remoteBlobProvider: pool, logger });\n const cachingStore = await HttpHelpers.createCachingObjectStore({\n upstream: store,\n config: cacheConfig,\n logger,\n });\n\n const handler = HttpHelpers.createRequestHandler({ store: cachingStore });\n const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n return new RemoteBlobProviderImpl(pool, server, cachingStore);\n }\n\n public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n return this.pool.acquire(params);\n }\n\n public httpServerInfo(): PFrameInternal.HttpServerInfo {\n return this.server.info;\n }\n\n public async getCacheMetrics(): Promise<PFrameInternal.CacheMetrics | null> {\n return this.cache.getMetrics();\n }\n\n public async resetCache(): Promise<void> {\n await this.cache.reset();\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.server.stop();\n await this.cache[Symbol.asyncDispose]();\n await this.pool[Symbol.asyncDispose]();\n }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n /** Port to run parquet HTTP server on. */\n parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n ...AbstractPFrameDriverOpsDefaults,\n parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\n/**\n * Tuning for the persistent parquet blob cache, minus its filesystem path (which\n * is supplied separately - see {@link createPFrameDriver}'s `cachePath`). Carries\n * only the size/eviction knobs of {@link PFrameInternal.CacheConfig}.\n */\nexport type ParquetCacheOps = Omit<PFrameInternal.CacheConfig, \"cachePath\">;\n\nexport async function createPFrameDriver(params: {\n blobDriver: DownloadDriver;\n logger: MiLogger;\n spillPath: string;\n cachePath: string;\n options: PFrameDriverOps;\n cacheOps: ParquetCacheOps;\n}): Promise<InternalPFrameDriver> {\n const resolvedSpillPath = path.resolve(params.spillPath);\n await emptyDir(resolvedSpillPath);\n\n const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n const remoteBlobProvider = await RemoteBlobProviderImpl.init(\n params.blobDriver,\n logger,\n { port: params.options.parquetServerPort },\n { cachePath: path.resolve(params.cachePath), ...params.cacheOps },\n );\n\n const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n if (isPlTreeNodeAccessor(data)) {\n return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n }\n\n return isDataInfo(data)\n ? data.type === \"ParquetPartitioned\"\n ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n : makeJsonDataInfo(spec, data);\n };\n\n return new AbstractPFrameDriver({\n logger,\n localBlobProvider,\n remoteBlobProvider,\n spillPath: resolvedSpillPath,\n options: params.options,\n resolveDataInfo,\n });\n}\n"],"mappings":";;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,aAAa,sBAAsB,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACU,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,CAAC,aAAa,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,iBAAiB,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEzE;;;AAKL,IAAM,iBAAN,cAA6B,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAI,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwB,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAAC,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,sEAAsE,YAAY,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,0DAA0D,YAAY,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAM,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,CAAC,aAAa,MAAM,CACtB,KAAI,0BAA0B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,wCAAwC,YAAY,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,uCAAuC,YAAY,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA,OACA;AAHiB,OAAA,OAAA;AACA,OAAA,SAAA;AACA,OAAA,QAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACA,aACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EACjE,MAAM,eAAe,MAAM,YAAY,yBAAyB;GAC9D,UAAU;GACV,QAAQ;GACR;GACD,CAAC;EAEF,MAAM,UAAU,YAAY,qBAAqB,EAAE,OAAO,cAAc,CAAC;EACzE,MAAM,SAAS,MAAM,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,QAAQ,aAAa;;CAG/D,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,MAAa,kBAA+D;AAC1E,SAAO,KAAK,MAAM,YAAY;;CAGhC,MAAa,aAA4B;AACvC,QAAM,KAAK,MAAM,OAAO;;CAG1B,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,MAAM,OAAO,eAAe;AACvC,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAG;CACH,mBAAmB;CACpB;AASD,eAAsB,mBAAmB,QAOP;CAChC,MAAM,oBAAoB,KAAK,QAAQ,OAAO,UAAU;AACxD,OAAM,SAAS,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KACtD,OAAO,YACP,QACA,EAAE,MAAM,OAAO,QAAQ,mBAAmB,EAC1C;EAAE,WAAW,KAAK,QAAQ,OAAO,UAAU;EAAE,GAAG,OAAO;EAAU,CAClE;CAED,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,MAAI,qBAAqB,KAAK,CAC5B,QAAO,sBAAsB,KAAK,IAAI,iBAAiB,MAAM,EAAE,CAAC;AAGlE,SAAO,WAAW,KAAK,GACnB,KAAK,SAAS,uBACZ,YAAY,OAAO,MAAM,6BAA6B,EAAE,CAAC,GACzD,YAAY,OAAO,MAAM,iBAAiB,EAAE,CAAC,GAC/C,iBAAiB,MAAM,KAAK;;AAGlC,QAAO,IAAI,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
|
|
1
|
+
{"version":3,"file":"driver.js","names":[],"sources":["../../src/pool/driver.ts"],"sourcesContent":["import {\n mapDataInfo,\n isDataInfo,\n ensureError,\n PFrameDriverError,\n isAbortError,\n type LocalBlobHandleAndSize,\n type RemoteBlobHandleAndSize,\n type RemoteBlobHandle,\n type ContentHandler,\n type PColumnSpec,\n type PColumnDataUniversal,\n} from \"@platforma-sdk/model\";\nimport { PFrameInternal } from \"@milaboratories/pl-model-middle-layer\";\nimport { emptyDir } from \"@milaboratories/ts-helpers\";\nimport { RefCountPoolBase, type PoolEntry, type MiLogger } from \"@milaboratories/helpers\";\nimport type { DownloadDriver } from \"@milaboratories/pl-drivers\";\nimport { isPlTreeNodeAccessor, type PlTreeNodeAccessor } from \"@milaboratories/pl-tree\";\nimport type { Computable, ComputableStableDefined } from \"@milaboratories/computable\";\nimport {\n makeJsonDataInfo,\n AbstractPFrameDriver,\n AbstractPFrameDriverOpsDefaults,\n type AbstractInternalPFrameDriver,\n type AbstractPFrameDriverOps,\n type LocalBlobProvider,\n type RemoteBlobProvider,\n} from \"@milaboratories/pf-driver\";\nimport { HttpHelpers } from \"@milaboratories/pframes-rs-node\";\nimport path from \"node:path\";\nimport { Readable } from \"node:stream\";\nimport {\n BlobResourceRef,\n makeLocalBlobRef,\n parseDataInfoResource,\n traverseParquetChunkResource,\n} from \"./data\";\nimport { isDownloadNetworkError400 } from \"@milaboratories/pl-drivers\";\nimport { parseSignedResourceId } from \"@milaboratories/pl-client\";\n\nfunction makeBlobId(res: BlobResourceRef): PFrameInternal.PFrameBlobId {\n const { globalId } = parseSignedResourceId(res.resourceInfo.id);\n return String(globalId) as PFrameInternal.PFrameBlobId;\n}\n\ntype LocalBlob = ComputableStableDefined<LocalBlobHandleAndSize>;\nclass LocalBlobProviderImpl\n extends RefCountPoolBase<BlobResourceRef, PFrameInternal.PFrameBlobId, LocalBlob>\n implements LocalBlobProvider<BlobResourceRef>\n{\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): LocalBlob {\n return this.blobDriver.getDownloadedBlob(params.resourceInfo);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): LocalBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Local blob with id ${blobId} not found.`);\n return resource;\n }\n\n public makeDataSource(\n signal: AbortSignal,\n ): Omit<PFrameInternal.PFrameDataSourceV2, \"parquetServer\"> {\n return {\n preloadBlob: async (blobIds: PFrameInternal.PFrameBlobId[]) => {\n try {\n await Promise.all(\n blobIds.map((blobId) => this.getByKey(blobId).awaitStableFullValue(signal)),\n );\n } catch (err: unknown) {\n if (!isAbortError(err)) throw err;\n }\n },\n resolveBlobContent: async (blobId: PFrameInternal.PFrameBlobId) => {\n const computable = this.getByKey(blobId);\n const blob = await computable.awaitStableValue(signal);\n return await this.blobDriver.getContentDirect(blob.handle, { signal });\n },\n };\n }\n}\n\ntype RemoteBlob = Computable<RemoteBlobHandleAndSize>;\nclass RemoteBlobPool extends RefCountPoolBase<\n BlobResourceRef,\n PFrameInternal.PFrameBlobId,\n RemoteBlob\n> {\n constructor(\n private readonly blobDriver: DownloadDriver,\n private readonly logger: PFrameInternal.Logger,\n ) {\n super();\n }\n\n protected calculateParamsKey(params: BlobResourceRef): PFrameInternal.PFrameBlobId {\n return makeBlobId(params);\n }\n\n protected createNewResource(\n params: BlobResourceRef,\n _key: PFrameInternal.PFrameBlobId,\n ): RemoteBlob {\n if (params.onDemandSnapshot === undefined) {\n throw new PFrameDriverError(\n `BlobResourceRef for rid ${params.toJSON()} is missing the on-demand snapshot; ` +\n `remote (parquet) blobs must be captured via makeRemoteBlobRef.`,\n );\n }\n return this.blobDriver.getOnDemandBlob(params.onDemandSnapshot);\n }\n\n public getByKey(blobId: PFrameInternal.PFrameBlobId): RemoteBlob {\n const resource = super.tryGetByKey(blobId);\n if (!resource) throw new PFrameDriverError(`Remote blob with id ${blobId} not found.`);\n return resource;\n }\n\n public async withContent<T>(\n handle: RemoteBlobHandle,\n options: {\n range: PFrameInternal.FileRange;\n signal: AbortSignal;\n handler: ContentHandler<T>;\n },\n ): Promise<T> {\n return await this.blobDriver.withContent(handle, {\n range: {\n from: options.range.start,\n to: options.range.end + 1,\n },\n signal: options.signal,\n handler: options.handler,\n });\n }\n}\n\ninterface BlobStoreOptions extends PFrameInternal.ObjectStoreOptions {\n remoteBlobProvider: RemoteBlobPool;\n}\n\nclass BlobStore extends PFrameInternal.BaseObjectStore {\n private readonly remoteBlobProvider: RemoteBlobPool;\n\n constructor(options: BlobStoreOptions) {\n super(options);\n this.remoteBlobProvider = options.remoteBlobProvider;\n }\n\n public override async request(\n filename: PFrameInternal.ParquetFileName,\n params: {\n method: PFrameInternal.HttpMethod;\n range?: PFrameInternal.HttpRange;\n signal: AbortSignal;\n callback: (response: PFrameInternal.ObjectStoreResponse) => Promise<void>;\n },\n ): Promise<void> {\n const blobId = filename.slice(\n 0,\n -PFrameInternal.ParquetExtension.length,\n ) as PFrameInternal.PFrameBlobId;\n const respond = async (response: PFrameInternal.ObjectStoreResponse): Promise<void> => {\n try {\n await params.callback(response);\n } catch (error: unknown) {\n this.logger(\n \"warn\",\n `PFrames blob store received unhandled rejection from HTTP handler: ${ensureError(error)}`,\n );\n }\n };\n\n try {\n const computable = this.remoteBlobProvider.tryGetByKey(blobId);\n if (!computable) return await respond({ type: \"NotFound\" });\n\n let blob: RemoteBlobHandleAndSize;\n try {\n blob = await computable.getValue();\n } catch (error: unknown) {\n this.logger(\n \"error\",\n `PFrames blob store failed to get blob from computable: ${ensureError(error)}`,\n );\n return await respond({ type: \"InternalError\" });\n }\n params.signal.throwIfAborted();\n\n const translatedRange = this.translate(blob.size, params.range);\n if (!translatedRange) {\n return await respond({\n type: \"RangeNotSatisfiable\",\n size: blob.size,\n });\n }\n\n if (params.method === \"HEAD\") {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n });\n }\n\n this.logger(\n \"info\",\n `PFrames blob store requesting content for ${blobId}, ` +\n `range [${translatedRange.start}..=${translatedRange.end}]`,\n );\n return await this.remoteBlobProvider.withContent(blob.handle, {\n range: translatedRange,\n signal: params.signal,\n handler: async (data) => {\n return await respond({\n type: \"Ok\",\n size: blob.size,\n range: translatedRange,\n data: Readable.fromWeb(data),\n });\n },\n });\n } catch (error: unknown) {\n if (!isAbortError(error)) {\n if (isDownloadNetworkError400(error) && error.statusCode === 404) {\n this.logger(\"info\", `PFrames blob store known race error: ${ensureError(error)}`);\n } else {\n this.logger(\"warn\", `PFrames blob store unhandled error: ${ensureError(error)}`);\n }\n }\n return await respond({ type: \"InternalError\" });\n }\n }\n}\n\nclass RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {\n constructor(\n private readonly pool: RemoteBlobPool,\n private readonly server: PFrameInternal.HttpServer,\n private readonly cache: PFrameInternal.CachingObjectStore,\n ) {}\n\n public static async init(\n blobDriver: DownloadDriver,\n logger: PFrameInternal.Logger,\n serverOptions: Omit<PFrameInternal.HttpServerOptions, \"handler\">,\n cacheConfig: PFrameInternal.CacheConfig,\n ): Promise<RemoteBlobProviderImpl> {\n const pool = new RemoteBlobPool(blobDriver, logger);\n const store = new BlobStore({ remoteBlobProvider: pool, logger });\n const cachingStore = (await HttpHelpers.createCachingObjectStore({\n upstream: store,\n config: cacheConfig,\n logger,\n })) as PFrameInternal.CachingObjectStore; // TODO: remove cast after next PFrames release\n\n const handler = HttpHelpers.createRequestHandler({ store: cachingStore });\n const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });\n logger(\"info\", `PFrames HTTP server started on ${server.info.url}`);\n\n return new RemoteBlobProviderImpl(pool, server, cachingStore);\n }\n\n public acquire(params: BlobResourceRef): PoolEntry<PFrameInternal.PFrameBlobId> {\n return this.pool.acquire(params);\n }\n\n public httpServerInfo(): PFrameInternal.HttpServerInfo {\n return this.server.info;\n }\n\n public async getCacheMetrics(): Promise<PFrameInternal.CacheMetrics | null> {\n return this.cache.getMetrics();\n }\n\n public async resetCache(): Promise<void> {\n await this.cache.reset();\n }\n\n async [Symbol.asyncDispose](): Promise<void> {\n await this.server.stop();\n await this.cache[Symbol.asyncDispose]();\n await this.pool[Symbol.asyncDispose]();\n }\n}\n\nexport interface InternalPFrameDriver extends AbstractInternalPFrameDriver<\n PColumnDataUniversal<PlTreeNodeAccessor>\n> {}\n\nexport type PFrameDriverOps = AbstractPFrameDriverOps & {\n /** Port to run parquet HTTP server on. */\n parquetServerPort: number;\n};\n\nexport const PFrameDriverOpsDefaults: PFrameDriverOps = {\n ...AbstractPFrameDriverOpsDefaults,\n parquetServerPort: 0, // 0 means that some unused port will be assigned by the OS\n};\n\n/**\n * Tuning for the persistent parquet blob cache, minus its filesystem path (which\n * is supplied separately - see {@link createPFrameDriver}'s `cachePath`). Carries\n * only the size/eviction knobs of {@link PFrameInternal.CacheConfig}.\n */\nexport type ParquetCacheOps = Omit<PFrameInternal.CacheConfig, \"cachePath\">;\n\nexport async function createPFrameDriver(params: {\n blobDriver: DownloadDriver;\n logger: MiLogger;\n spillPath: string;\n cachePath: string;\n options: PFrameDriverOps;\n cacheOps: ParquetCacheOps;\n}): Promise<InternalPFrameDriver> {\n const resolvedSpillPath = path.resolve(params.spillPath);\n await emptyDir(resolvedSpillPath);\n\n const logger: PFrameInternal.Logger = (level, message) => params.logger[level](message);\n const localBlobProvider = new LocalBlobProviderImpl(params.blobDriver, logger);\n const remoteBlobProvider = await RemoteBlobProviderImpl.init(\n params.blobDriver,\n logger,\n { port: params.options.parquetServerPort },\n { cachePath: path.resolve(params.cachePath), ...params.cacheOps },\n );\n\n const resolveDataInfo = (spec: PColumnSpec, data: PColumnDataUniversal<PlTreeNodeAccessor>) => {\n if (isPlTreeNodeAccessor(data)) {\n return parseDataInfoResource(data) ?? makeJsonDataInfo(spec, []);\n }\n\n return isDataInfo(data)\n ? data.type === \"ParquetPartitioned\"\n ? mapDataInfo(data, (a) => traverseParquetChunkResource(a))\n : mapDataInfo(data, (a) => makeLocalBlobRef(a))\n : makeJsonDataInfo(spec, data);\n };\n\n return new AbstractPFrameDriver({\n logger,\n localBlobProvider,\n remoteBlobProvider,\n spillPath: resolvedSpillPath,\n options: params.options,\n resolveDataInfo,\n });\n}\n"],"mappings":";;;;;;;;;;;;;AAwCA,SAAS,WAAW,KAAmD;CACrE,MAAM,EAAE,aAAa,sBAAsB,IAAI,aAAa,GAAG;AAC/D,QAAO,OAAO,SAAS;;AAIzB,IAAM,wBAAN,cACU,iBAEV;CACE,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACW;AACX,SAAO,KAAK,WAAW,kBAAkB,OAAO,aAAa;;CAG/D,SAAgB,QAAgD;EAC9D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,sBAAsB,OAAO,aAAa;AACrF,SAAO;;CAGT,eACE,QAC0D;AAC1D,SAAO;GACL,aAAa,OAAO,YAA2C;AAC7D,QAAI;AACF,WAAM,QAAQ,IACZ,QAAQ,KAAK,WAAW,KAAK,SAAS,OAAO,CAAC,qBAAqB,OAAO,CAAC,CAC5E;aACM,KAAc;AACrB,SAAI,CAAC,aAAa,IAAI,CAAE,OAAM;;;GAGlC,oBAAoB,OAAO,WAAwC;IAEjE,MAAM,OAAO,MADM,KAAK,SAAS,OAAO,CACV,iBAAiB,OAAO;AACtD,WAAO,MAAM,KAAK,WAAW,iBAAiB,KAAK,QAAQ,EAAE,QAAQ,CAAC;;GAEzE;;;AAKL,IAAM,iBAAN,cAA6B,iBAI3B;CACA,YACE,YACA,QACA;AACA,SAAO;AAHU,OAAA,aAAA;AACA,OAAA,SAAA;;CAKnB,mBAA6B,QAAsD;AACjF,SAAO,WAAW,OAAO;;CAG3B,kBACE,QACA,MACY;AACZ,MAAI,OAAO,qBAAqB,KAAA,EAC9B,OAAM,IAAI,kBACR,2BAA2B,OAAO,QAAQ,CAAC,oGAE5C;AAEH,SAAO,KAAK,WAAW,gBAAgB,OAAO,iBAAiB;;CAGjE,SAAgB,QAAiD;EAC/D,MAAM,WAAW,MAAM,YAAY,OAAO;AAC1C,MAAI,CAAC,SAAU,OAAM,IAAI,kBAAkB,uBAAuB,OAAO,aAAa;AACtF,SAAO;;CAGT,MAAa,YACX,QACA,SAKY;AACZ,SAAO,MAAM,KAAK,WAAW,YAAY,QAAQ;GAC/C,OAAO;IACL,MAAM,QAAQ,MAAM;IACpB,IAAI,QAAQ,MAAM,MAAM;IACzB;GACD,QAAQ,QAAQ;GAChB,SAAS,QAAQ;GAClB,CAAC;;;AAQN,IAAM,YAAN,cAAwB,eAAe,gBAAgB;CACrD;CAEA,YAAY,SAA2B;AACrC,QAAM,QAAQ;AACd,OAAK,qBAAqB,QAAQ;;CAGpC,MAAsB,QACpB,UACA,QAMe;EACf,MAAM,SAAS,SAAS,MACtB,GACA,CAAC,eAAe,iBAAiB,OAClC;EACD,MAAM,UAAU,OAAO,aAAgE;AACrF,OAAI;AACF,UAAM,OAAO,SAAS,SAAS;YACxB,OAAgB;AACvB,SAAK,OACH,QACA,sEAAsE,YAAY,MAAM,GACzF;;;AAIL,MAAI;GACF,MAAM,aAAa,KAAK,mBAAmB,YAAY,OAAO;AAC9D,OAAI,CAAC,WAAY,QAAO,MAAM,QAAQ,EAAE,MAAM,YAAY,CAAC;GAE3D,IAAI;AACJ,OAAI;AACF,WAAO,MAAM,WAAW,UAAU;YAC3B,OAAgB;AACvB,SAAK,OACH,SACA,0DAA0D,YAAY,MAAM,GAC7E;AACD,WAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;AAEjD,UAAO,OAAO,gBAAgB;GAE9B,MAAM,kBAAkB,KAAK,UAAU,KAAK,MAAM,OAAO,MAAM;AAC/D,OAAI,CAAC,gBACH,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACZ,CAAC;AAGJ,OAAI,OAAO,WAAW,OACpB,QAAO,MAAM,QAAQ;IACnB,MAAM;IACN,MAAM,KAAK;IACX,OAAO;IACR,CAAC;AAGJ,QAAK,OACH,QACA,6CAA6C,OAAO,WACxC,gBAAgB,MAAM,KAAK,gBAAgB,IAAI,GAC5D;AACD,UAAO,MAAM,KAAK,mBAAmB,YAAY,KAAK,QAAQ;IAC5D,OAAO;IACP,QAAQ,OAAO;IACf,SAAS,OAAO,SAAS;AACvB,YAAO,MAAM,QAAQ;MACnB,MAAM;MACN,MAAM,KAAK;MACX,OAAO;MACP,MAAM,SAAS,QAAQ,KAAK;MAC7B,CAAC;;IAEL,CAAC;WACK,OAAgB;AACvB,OAAI,CAAC,aAAa,MAAM,CACtB,KAAI,0BAA0B,MAAM,IAAI,MAAM,eAAe,IAC3D,MAAK,OAAO,QAAQ,wCAAwC,YAAY,MAAM,GAAG;OAEjF,MAAK,OAAO,QAAQ,uCAAuC,YAAY,MAAM,GAAG;AAGpF,UAAO,MAAM,QAAQ,EAAE,MAAM,iBAAiB,CAAC;;;;AAKrD,IAAM,yBAAN,MAAM,uBAAsE;CAC1E,YACE,MACA,QACA,OACA;AAHiB,OAAA,OAAA;AACA,OAAA,SAAA;AACA,OAAA,QAAA;;CAGnB,aAAoB,KAClB,YACA,QACA,eACA,aACiC;EACjC,MAAM,OAAO,IAAI,eAAe,YAAY,OAAO;EACnD,MAAM,QAAQ,IAAI,UAAU;GAAE,oBAAoB;GAAM;GAAQ,CAAC;EACjE,MAAM,eAAgB,MAAM,YAAY,yBAAyB;GAC/D,UAAU;GACV,QAAQ;GACR;GACD,CAAC;EAEF,MAAM,UAAU,YAAY,qBAAqB,EAAE,OAAO,cAAc,CAAC;EACzE,MAAM,SAAS,MAAM,YAAY,iBAAiB;GAAE,GAAG;GAAe;GAAS,CAAC;AAChF,SAAO,QAAQ,kCAAkC,OAAO,KAAK,MAAM;AAEnE,SAAO,IAAI,uBAAuB,MAAM,QAAQ,aAAa;;CAG/D,QAAe,QAAiE;AAC9E,SAAO,KAAK,KAAK,QAAQ,OAAO;;CAGlC,iBAAuD;AACrD,SAAO,KAAK,OAAO;;CAGrB,MAAa,kBAA+D;AAC1E,SAAO,KAAK,MAAM,YAAY;;CAGhC,MAAa,aAA4B;AACvC,QAAM,KAAK,MAAM,OAAO;;CAG1B,OAAO,OAAO,gBAA+B;AAC3C,QAAM,KAAK,OAAO,MAAM;AACxB,QAAM,KAAK,MAAM,OAAO,eAAe;AACvC,QAAM,KAAK,KAAK,OAAO,eAAe;;;AAa1C,MAAa,0BAA2C;CACtD,GAAG;CACH,mBAAmB;CACpB;AASD,eAAsB,mBAAmB,QAOP;CAChC,MAAM,oBAAoB,KAAK,QAAQ,OAAO,UAAU;AACxD,OAAM,SAAS,kBAAkB;CAEjC,MAAM,UAAiC,OAAO,YAAY,OAAO,OAAO,OAAO,QAAQ;CACvF,MAAM,oBAAoB,IAAI,sBAAsB,OAAO,YAAY,OAAO;CAC9E,MAAM,qBAAqB,MAAM,uBAAuB,KACtD,OAAO,YACP,QACA,EAAE,MAAM,OAAO,QAAQ,mBAAmB,EAC1C;EAAE,WAAW,KAAK,QAAQ,OAAO,UAAU;EAAE,GAAG,OAAO;EAAU,CAClE;CAED,MAAM,mBAAmB,MAAmB,SAAmD;AAC7F,MAAI,qBAAqB,KAAK,CAC5B,QAAO,sBAAsB,KAAK,IAAI,iBAAiB,MAAM,EAAE,CAAC;AAGlE,SAAO,WAAW,KAAK,GACnB,KAAK,SAAS,uBACZ,YAAY,OAAO,MAAM,6BAA6B,EAAE,CAAC,GACzD,YAAY,OAAO,MAAM,iBAAiB,EAAE,CAAC,GAC/C,iBAAiB,MAAM,KAAK;;AAGlC,QAAO,IAAI,qBAAqB;EAC9B;EACA;EACA;EACA,WAAW;EACX,SAAS,OAAO;EAChB;EACD,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@milaboratories/pl-middle-layer",
|
|
3
|
-
"version": "1.64.
|
|
3
|
+
"version": "1.64.35",
|
|
4
4
|
"description": "Pl Middle Layer",
|
|
5
5
|
"keywords": [],
|
|
6
6
|
"license": "UNLICENSED",
|
|
@@ -19,8 +19,8 @@
|
|
|
19
19
|
}
|
|
20
20
|
},
|
|
21
21
|
"dependencies": {
|
|
22
|
-
"@milaboratories/pframes-rs-node": "1.1.
|
|
23
|
-
"@milaboratories/pframes-rs-wasm": "1.1.
|
|
22
|
+
"@milaboratories/pframes-rs-node": "1.1.52",
|
|
23
|
+
"@milaboratories/pframes-rs-wasm": "1.1.52",
|
|
24
24
|
"canonicalize": "~2.1.0",
|
|
25
25
|
"denque": "^2.1.0",
|
|
26
26
|
"es-toolkit": "^1.39.10",
|
|
@@ -32,23 +32,23 @@
|
|
|
32
32
|
"yaml": "^2.8.0",
|
|
33
33
|
"zod": "~3.25.76",
|
|
34
34
|
"@milaboratories/computable": "2.9.5",
|
|
35
|
+
"@milaboratories/pf-spec-driver": "1.4.15",
|
|
36
|
+
"@milaboratories/pf-driver": "1.7.13",
|
|
35
37
|
"@milaboratories/helpers": "1.14.2",
|
|
36
|
-
"@milaboratories/pf-driver": "1.7.12",
|
|
37
|
-
"@milaboratories/pl-deployments": "3.0.8",
|
|
38
38
|
"@milaboratories/pl-client": "3.11.5",
|
|
39
|
-
"@milaboratories/pf-spec-driver": "1.4.14",
|
|
40
39
|
"@milaboratories/pl-drivers": "1.16.1",
|
|
40
|
+
"@milaboratories/pl-deployments": "3.0.8",
|
|
41
41
|
"@milaboratories/pl-errors": "1.4.23",
|
|
42
|
+
"@milaboratories/pl-http": "1.2.4",
|
|
42
43
|
"@milaboratories/pl-model-backend": "1.4.8",
|
|
44
|
+
"@milaboratories/pl-model-middle-layer": "1.30.8",
|
|
43
45
|
"@milaboratories/pl-model-common": "1.46.2",
|
|
44
|
-
"@milaboratories/pl-http": "1.2.4",
|
|
45
|
-
"@milaboratories/pl-model-middle-layer": "1.30.7",
|
|
46
|
-
"@milaboratories/pl-tree": "1.12.13",
|
|
47
46
|
"@milaboratories/resolve-helper": "1.1.3",
|
|
48
47
|
"@milaboratories/ts-helpers": "1.8.3",
|
|
49
|
-
"@
|
|
50
|
-
"@platforma-sdk/model": "1.79.
|
|
51
|
-
"@platforma-sdk/
|
|
48
|
+
"@milaboratories/pl-tree": "1.12.13",
|
|
49
|
+
"@platforma-sdk/model": "1.79.17",
|
|
50
|
+
"@platforma-sdk/block-tools": "2.11.2",
|
|
51
|
+
"@platforma-sdk/workflow-tengo": "6.6.4"
|
|
52
52
|
},
|
|
53
53
|
"devDependencies": {
|
|
54
54
|
"@types/node": "~24.5.2",
|
|
@@ -56,8 +56,8 @@
|
|
|
56
56
|
"@vitest/coverage-istanbul": "^4.1.3",
|
|
57
57
|
"typescript": "~5.9.3",
|
|
58
58
|
"vitest": "^4.1.3",
|
|
59
|
-
"@milaboratories/build-configs": "2.0.0",
|
|
60
59
|
"@milaboratories/ts-configs": "1.2.3",
|
|
60
|
+
"@milaboratories/build-configs": "2.0.0",
|
|
61
61
|
"@milaboratories/ts-builder": "1.5.2"
|
|
62
62
|
},
|
|
63
63
|
"engines": {
|
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
PlTemplateOverrideV1,
|
|
19
19
|
PlTemplateSoftwareV1,
|
|
20
20
|
PlTemplateV1,
|
|
21
|
+
PlWasmV1,
|
|
21
22
|
} from "@milaboratories/pl-model-backend";
|
|
22
23
|
import type {
|
|
23
24
|
CompiledTemplateV3,
|
|
@@ -27,6 +28,7 @@ import type {
|
|
|
27
28
|
TemplateLibDataV3,
|
|
28
29
|
TemplateSoftwareData,
|
|
29
30
|
TemplateSoftwareDataV3,
|
|
31
|
+
TemplateWasmDataV3,
|
|
30
32
|
} from "@milaboratories/pl-model-backend";
|
|
31
33
|
import { notEmpty } from "@milaboratories/ts-helpers";
|
|
32
34
|
import type { BlockPackSpecPrepared } from "../../model";
|
|
@@ -150,6 +152,16 @@ function hashSoftwareV3(sw: TemplateSoftwareDataV3): string {
|
|
|
150
152
|
.digest("hex");
|
|
151
153
|
}
|
|
152
154
|
|
|
155
|
+
function hashWasmV3(wasm: TemplateWasmDataV3): string {
|
|
156
|
+
return createHash("sha256")
|
|
157
|
+
.update(PlWasmV1.type.name)
|
|
158
|
+
.update(PlWasmV1.type.version)
|
|
159
|
+
.update(wasm.name)
|
|
160
|
+
.update(wasm.version)
|
|
161
|
+
.update(wasm.sourceHash)
|
|
162
|
+
.digest("hex");
|
|
163
|
+
}
|
|
164
|
+
|
|
153
165
|
// ─── Tree flattening ─────────────────────────────────────────────────────────
|
|
154
166
|
|
|
155
167
|
function flattenV2Tree(data: TemplateData): CacheableNode[] {
|
|
@@ -316,6 +328,25 @@ function flattenV3Tree(data: CompiledTemplateV3): CacheableNode[] {
|
|
|
316
328
|
return hash;
|
|
317
329
|
}
|
|
318
330
|
|
|
331
|
+
function processWasm(wasm: TemplateWasmDataV3): string {
|
|
332
|
+
const hash = hashWasmV3(wasm);
|
|
333
|
+
if (!seen.has(hash)) {
|
|
334
|
+
seen.add(hash);
|
|
335
|
+
nodes.push({
|
|
336
|
+
hash,
|
|
337
|
+
create: (tx) =>
|
|
338
|
+
tx.createValue(
|
|
339
|
+
PlWasmV1.type,
|
|
340
|
+
JSON.stringify(
|
|
341
|
+
PlWasmV1.fromV3Data(wasm, getSourceCode(wasm.name, sources, wasm.sourceHash)).data,
|
|
342
|
+
),
|
|
343
|
+
),
|
|
344
|
+
childHashes: [],
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
return hash;
|
|
348
|
+
}
|
|
349
|
+
|
|
319
350
|
function processTemplate(tpl: TemplateDataV3): string {
|
|
320
351
|
// Process children first (bottom-up)
|
|
321
352
|
const childHashes: string[] = [];
|
|
@@ -341,6 +372,11 @@ function flattenV3Tree(data: CompiledTemplateV3): CacheableNode[] {
|
|
|
341
372
|
childHashes.push(h);
|
|
342
373
|
children.push({ fieldName: `${PlTemplateV1.tplPrefix}/${tplId}`, hash: h });
|
|
343
374
|
}
|
|
375
|
+
for (const [wasmId, wasm] of Object.entries(tpl.wasm ?? {})) {
|
|
376
|
+
const h = processWasm(wasm);
|
|
377
|
+
childHashes.push(h);
|
|
378
|
+
children.push({ fieldName: `${PlTemplateV1.wasmPrefix}/${wasmId}`, hash: h });
|
|
379
|
+
}
|
|
344
380
|
|
|
345
381
|
// Compose hash from own content + child hash strings (NOT child content).
|
|
346
382
|
// Uses sourceHash directly — it already uniquely identifies the source.
|
package/src/pool/driver.ts
CHANGED
|
@@ -262,11 +262,11 @@ class RemoteBlobProviderImpl implements RemoteBlobProvider<BlobResourceRef> {
|
|
|
262
262
|
): Promise<RemoteBlobProviderImpl> {
|
|
263
263
|
const pool = new RemoteBlobPool(blobDriver, logger);
|
|
264
264
|
const store = new BlobStore({ remoteBlobProvider: pool, logger });
|
|
265
|
-
const cachingStore = await HttpHelpers.createCachingObjectStore({
|
|
265
|
+
const cachingStore = (await HttpHelpers.createCachingObjectStore({
|
|
266
266
|
upstream: store,
|
|
267
267
|
config: cacheConfig,
|
|
268
268
|
logger,
|
|
269
|
-
});
|
|
269
|
+
})) as PFrameInternal.CachingObjectStore; // TODO: remove cast after next PFrames release
|
|
270
270
|
|
|
271
271
|
const handler = HttpHelpers.createRequestHandler({ store: cachingStore });
|
|
272
272
|
const server = await HttpHelpers.createHttpServer({ ...serverOptions, handler });
|