@query-farm/vgi 0.38.0 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/client.d.ts +31 -2
- package/dist/client/types.d.ts +56 -1
- package/dist/client-entry.js +99 -8
- package/dist/client-entry.js.map +4 -4
- package/dist/functions/copy-to.d.ts +1 -1
- package/dist/index.core.d.ts +1 -1
- package/dist/index.js +99 -8
- package/dist/index.js.map +6 -6
- package/dist/serve-entry.js.map +1 -1
- package/dist/worker-cf-entry.js.map +3 -3
- package/package.json +1 -1
- package/src/client/client.ts +148 -0
- package/src/client/types.ts +58 -1
- package/src/codec/asd.ts +12 -8
- package/src/functions/copy-to.ts +1 -1
- package/src/index.core.ts +2 -0
package/dist/serve-entry.js.map
CHANGED
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
"// Copyright 2025, 2026 Query Farm LLC - https://query.farm\n// Handlers for the aggregate function lifecycle: bind allocates execution_id\n// and output schema, update folds rows into per-group state, combine merges\n// states across workers, finalize emits one result row per group, destructor\n// drops finished group state.\n\nimport {\n type VgiSchema,\n type VgiBatch,\n type VgiDataType,\n schema as makeSchema,\n field,\n isDecimal,\n serializeBatch,\n deserializeBatch,\n} from \"../../arrow/index.js\";\nimport { Protocol } from \"@query-farm/vgi-rpc\";\nimport type { FunctionRegistry } from \"../../functions/registry.js\";\nimport { deserializeArguments } from \"../serialize.js\";\nimport { deserializeSchema, serializeSchema, batchToSecretDict } from \"../../util/arrow/index.js\";\nimport { toUint8Array } from \"../../util/bytes.js\";\nimport {\n AggregateBindParamsSchema,\n AggregateBindResultSchema,\n AggregateCombineParamsSchema,\n AggregateDestructorParamsSchema,\n AggregateFinalizeParamsSchema,\n AggregateFinalizeResultSchema,\n AggregateUpdateParamsSchema,\n} from \"../../generated/vgi-protocol-schemas.js\";\nimport {\n getExecutionState,\n setExecutionState,\n persistGroupStates,\n GROUP_COLUMN_NAME,\n type AggregateFunctionConfig,\n type AggregateBindParams,\n} from \"../../functions/aggregate.js\";\nimport { Arguments } from \"../../arguments/arguments.js\";\nimport { validateConstConstraints } from \"../../arguments/argument-spec.js\";\nimport {\n REQUEST_PARAMS_SCHEMA,\n RESULT_BINARY_SCHEMA,\n unwrapRequest,\n wrapResult,\n} from \"./shared.js\";\n\nexport function registerAggregateMethods(protocol: Protocol, registry: FunctionRegistry): void {\n // `schemaPath` scopes resolution to the schema the caller named. Every\n // aggregate RPC carries it (protocol 1.2.0), so an aggregate name declared in\n // two schemas dispatches to the one the request names rather than colliding\n // on the flat by-name index. `null` (no schema) keeps the cross-schema lookup.\n function resolveAggregate(\n name: string,\n schemaPath?: string[] | null,\n ): AggregateFunctionConfig<any, any> {\n const func = registry.get(name, {\n arguments: new Arguments(),\n inputSchema: null,\n isScalar: false,\n schemaPath: schemaPath ?? null,\n }) as any;\n if (!func || func.kind !== \"aggregate\") {\n throw new Error(`Function '${name}' is not an aggregate function`);\n }\n return func.aggregateConfig as AggregateFunctionConfig<any, any>;\n }\n\n // ------------------------------------------------------------------------\n // aggregate_bind — allocate execution_id, return output schema\n // ------------------------------------------------------------------------\n protocol.unary(\"aggregate_bind\", {\n params: AggregateBindParamsSchema,\n result: RESULT_BINARY_SCHEMA,\n handler: async (params) => {\n const innerParams = unwrapRequest(params.request);\n const functionName: string = innerParams.function_name;\n const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);\n const argBytes = toUint8Array(innerParams.arguments);\n const args = deserializeArguments(argBytes);\n const inputSchema = innerParams.input_schema\n ? deserializeSchema(toUint8Array(innerParams.input_schema))\n : null;\n // Decode the resolved-secrets batch the extension pre-resolves for\n // aggregates that declared `requiredSecrets`. Mirrors the table-buffering\n // handler (batchToSecretDict). The secret VALUES are delivered only here,\n // at bind time — update/combine/finalize see no secrets by design.\n const secretsBytes = innerParams.secrets ? toUint8Array(innerParams.secrets) : null;\n const secretsBatch =\n secretsBytes && secretsBytes.length > 0 ? deserializeBatch(secretsBytes) : null;\n const secrets = secretsBatch ? batchToSecretDict(secretsBatch) : {};\n const bindParams: AggregateBindParams<any> = {\n args: extractArgMap(cfg, args),\n arguments: args,\n inputSchema,\n settings: {},\n secrets,\n argumentNames: innerParams.argument_names\n ? (Array.isArray(innerParams.argument_names)\n ? innerParams.argument_names\n : [...innerParams.argument_names])\n .map((value: unknown) => value == null ? null : String(value))\n : null,\n };\n const executionId = new Uint8Array(16);\n crypto.getRandomValues(executionId);\n setExecutionState(executionId, {\n states: new Map(),\n bindArgs: args,\n bindParams,\n });\n // Output schema is a single \"result\" column. Type comes from the\n // optional onBind hook (for aggregates whose return type depends on\n // input — e.g. vgi_generic_sum takes ANY and returns the same type);\n // otherwise the declared cfg.outputType.\n const outType = cfg.onBind ? await cfg.onBind(bindParams) : cfg.outputType;\n const outSchema = makeSchema([field(\"result\", outType, true)]);\n return wrapResult(\n { output_schema: serializeSchema(outSchema), execution_id: executionId },\n AggregateBindResultSchema,\n );\n },\n });\n\n // ------------------------------------------------------------------------\n // aggregate_update — fold the incoming batch into per-group state\n // ------------------------------------------------------------------------\n protocol.unary(\"aggregate_update\", {\n params: AggregateUpdateParamsSchema,\n result: RESULT_BINARY_SCHEMA,\n handler: (params) => {\n const innerParams = unwrapRequest(params.request);\n const functionName: string = innerParams.function_name;\n const executionId = toUint8Array(innerParams.execution_id);\n const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);\n const batch = readSingleBatch(toUint8Array(innerParams.input_batch));\n if (!batch || batch.numRows === 0) return wrapResult({}, makeSchema([]));\n\n const gidIdx = batch.schema.fields.findIndex((f) => f.name === GROUP_COLUMN_NAME);\n if (gidIdx < 0) {\n throw new Error(`aggregate_update: input batch missing ${GROUP_COLUMN_NAME} column`);\n }\n const gidCol = batch.getChildAt(gidIdx)!;\n const groupIds: bigint[] = [];\n const uniqueGids = new Set<bigint>();\n for (let i = 0; i < batch.numRows; i++) {\n const v = gidCol.get(i);\n const g = typeof v === \"bigint\" ? v : BigInt(v as any);\n groupIds.push(g);\n uniqueGids.add(g);\n }\n const exec = getExecutionState(executionId, uniqueGids);\n if (!exec) {\n throw new Error(`aggregate_update: unknown execution_id for '${functionName}' (bind missing or state dropped)`);\n }\n\n // Build column-arg arrays — every non-__vgi_group_id column, in\n // schema order — and pass to user's update(). We deliberately do NOT\n // pre-allocate state for every group_id seen — the user decides when\n // to call ensureState() so that groups with only-NULL input (skipped\n // rows) never get an entry, which lets finalize() return SQL NULL.\n const columns: any[] = [];\n for (let i = 0; i < batch.schema.fields.length; i++) {\n if (i === gidIdx) continue;\n columns.push(batch.getChildAt(i));\n }\n const ensureState = (gid: bigint) => {\n let s = exec.states.get(gid);\n if (s === undefined) {\n s = cfg.initialState(exec.bindParams);\n exec.states.set(gid, s);\n }\n return s;\n };\n cfg.update({\n states: exec.states,\n groupIds,\n columns,\n args: exec.bindParams.args,\n ensureState,\n });\n // Flush mutated states to disk so sibling workers see them. We persist\n // every group that was referenced this call — some may be unchanged\n // (skipped by NULL handling), in which case the file simply reflects\n // the pre-existing initial state; the idempotent rewrite is cheap\n // relative to the RPC overhead.\n persistGroupStates(executionId, uniqueGids);\n return wrapResult({}, makeSchema([]));\n },\n });\n\n // ------------------------------------------------------------------------\n // aggregate_combine — merge (source_group_id, target_group_id) pairs\n // ------------------------------------------------------------------------\n protocol.unary(\"aggregate_combine\", {\n params: AggregateCombineParamsSchema,\n result: RESULT_BINARY_SCHEMA,\n handler: (params) => {\n const innerParams = unwrapRequest(params.request);\n const functionName: string = innerParams.function_name;\n const executionId = toUint8Array(innerParams.execution_id);\n const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);\n const batch = readSingleBatch(toUint8Array(innerParams.merge_batch));\n if (!batch || batch.numRows === 0) return wrapResult({}, makeSchema([]));\n\n const srcCol = batch.getChild(\"source_group_id\")!;\n const tgtCol = batch.getChild(\"target_group_id\")!;\n const referencedGids = new Set<bigint>();\n const srcIds: bigint[] = [];\n const tgtIds: bigint[] = [];\n for (let i = 0; i < batch.numRows; i++) {\n const src = BigInt(srcCol.get(i) as any);\n const tgt = BigInt(tgtCol.get(i) as any);\n srcIds.push(src); tgtIds.push(tgt);\n referencedGids.add(src); referencedGids.add(tgt);\n }\n const exec = getExecutionState(executionId, referencedGids);\n if (!exec) {\n throw new Error(\n `aggregate_combine: unknown execution_id for '${functionName}' ` +\n `(context missing on disk — bind may have failed or been cleaned up)`,\n );\n }\n const touchedTargets = new Set<bigint>();\n for (let i = 0; i < batch.numRows; i++) {\n const src = srcIds[i];\n const tgt = tgtIds[i];\n const srcHas = exec.states.has(src);\n const tgtHas = exec.states.has(tgt);\n if (!srcHas && !tgtHas) continue;\n const srcState = srcHas ? exec.states.get(src)! : cfg.initialState(exec.bindParams);\n const tgtState = tgtHas ? exec.states.get(tgt)! : cfg.initialState(exec.bindParams);\n exec.states.set(tgt, cfg.combine(srcState, tgtState, exec.bindParams));\n touchedTargets.add(tgt);\n }\n persistGroupStates(executionId, touchedTargets);\n return wrapResult({}, makeSchema([]));\n },\n });\n\n // ------------------------------------------------------------------------\n // aggregate_finalize — emit one result row per requested group_id\n // ------------------------------------------------------------------------\n protocol.unary(\"aggregate_finalize\", {\n params: AggregateFinalizeParamsSchema,\n result: RESULT_BINARY_SCHEMA,\n handler: (params) => {\n const innerParams = unwrapRequest(params.request);\n const functionName: string = innerParams.function_name;\n const executionId = toUint8Array(innerParams.execution_id);\n const cfg = resolveAggregate(functionName, innerParams.schema_path ?? null);\n const outputSchema = deserializeSchema(toUint8Array(innerParams.output_schema));\n const gidBatch = readSingleBatch(toUint8Array(innerParams.group_ids_batch));\n const groupIds: bigint[] = [];\n const uniqueGids = new Set<bigint>();\n if (gidBatch) {\n const col = gidBatch.getChild(\"group_id\")!;\n for (let i = 0; i < gidBatch.numRows; i++) {\n const v = col.get(i);\n const g = typeof v === \"bigint\" ? v : BigInt(v as any);\n groupIds.push(g);\n uniqueGids.add(g);\n }\n }\n const exec = getExecutionState(executionId, uniqueGids);\n if (!exec) {\n throw new Error(`aggregate_finalize: unknown execution_id for '${functionName}'`);\n }\n // Groups never updated pass state = null so finalize() can emit SQL NULL\n // (matches Python semantics for SUM/AVG/MIN/MAX over zero rows).\n const states = new Map<bigint, any | null>();\n for (const gid of groupIds) {\n states.set(gid, exec.states.has(gid) ? exec.states.get(gid) : null);\n }\n const resultBatch = cfg.finalize({\n groupIds, states, outputSchema, args: exec.bindParams.args,\n });\n return wrapResult(\n { result_batch: serializeBatchWithSchema(resultBatch) },\n AggregateFinalizeResultSchema,\n );\n },\n });\n\n // ------------------------------------------------------------------------\n // aggregate_destructor — drop state for finished groups (best-effort)\n //\n // DuckDB sends a batch of group_ids to discard; we remove each from the\n // in-memory state map. We DO NOT tear down the entire execution on an\n // empty batch — DuckDB may still call finalize afterwards in some plans\n // (matches Python's conservative behavior; the execution entry is cleaned\n // up when the worker process exits).\n // ------------------------------------------------------------------------\n protocol.unary(\"aggregate_destructor\", {\n params: AggregateDestructorParamsSchema,\n result: RESULT_BINARY_SCHEMA,\n handler: (params) => {\n const innerParams = unwrapRequest(params.request);\n const executionId = toUint8Array(innerParams.execution_id);\n const exec = getExecutionState(executionId);\n if (!exec) return wrapResult({}, makeSchema([]));\n const gidBatch = readSingleBatch(toUint8Array(innerParams.group_ids_batch));\n if (!gidBatch || gidBatch.numRows === 0) return wrapResult({}, makeSchema([]));\n const col = gidBatch.getChild(\"group_id\")!;\n const toDrop = new Set<bigint>();\n for (let i = 0; i < gidBatch.numRows; i++) {\n const v = col.get(i);\n const g = typeof v === \"bigint\" ? v : BigInt(v as any);\n exec.states.delete(g);\n toDrop.add(g);\n }\n // Persist deletions (unlinks the per-group file so sibling workers\n // don't revive the state via getExecutionState's lazy disk load).\n persistGroupStates(executionId, toDrop);\n return wrapResult({}, makeSchema([]));\n },\n });\n}\n\nfunction extractArgMap(\n cfg: AggregateFunctionConfig<any, any>,\n args: Arguments,\n): Record<string, any> {\n const out: Record<string, any> = {};\n // The wire-side Arguments contains only const params — DuckDB erases const\n // args from the expression tree at bind time and serializes their values\n // into the aggregate_bind request's `arguments` field. So we index the\n // Arguments by the declaration order of const params only.\n const constSet = new Set(cfg.constParams ?? []);\n const constNames: string[] = [];\n if (cfg.args) {\n for (const name of Object.keys(cfg.args)) {\n if (constSet.has(name)) constNames.push(name);\n }\n }\n for (let i = 0; i < constNames.length; i++) {\n let v: any;\n try {\n v = args.get(i);\n // Decode DECIMAL scalars (DuckDB serializes 0.5-style literals as\n // DECIMAL(2,1), which Arrow stores as the unscaled integer — a raw\n // BigInt here). Scale back to float using the declared scale.\n const field = args.argumentsSchema?.fields.find(f => f.name === `positional_${i}`);\n if (field && isDecimal(field.type)) {\n const scale = (field.type as any).scale;\n if (typeof v === \"bigint\") v = Number(v) / Math.pow(10, scale);\n else if (typeof v === \"number\") v = v / Math.pow(10, scale);\n } else if (typeof v === \"bigint\") {\n v = Number(v);\n }\n } catch {\n // args.get throws when the arg isn't set — fall back to the declared\n // default so aggregates called without an explicit const value still\n // see it in update/finalize.\n const def = cfg.argDefaults?.[constNames[i]];\n if (def !== undefined) out[constNames[i]] = def;\n continue;\n }\n // Enforce declared constraints on the supplied value. This sits OUTSIDE the\n // try above so a constraint violation propagates as a bind error rather than\n // being swallowed and silently replaced by the default.\n const constraints = cfg.argConstraints?.[constNames[i]];\n if (constraints !== undefined) {\n validateConstConstraints(constNames[i], constraints, v);\n }\n out[constNames[i]] = v;\n }\n // Non-const args aren't in the wire Arguments, but users may still want\n // defaults surfaced for `params.args` — apply them here.\n if (cfg.argDefaults) {\n for (const [name, def] of Object.entries(cfg.argDefaults)) {\n if (!(name in out) && !constSet.has(name)) out[name] = def;\n }\n }\n return out;\n}\n\nfunction readSingleBatch(bytes: Uint8Array): VgiBatch | null {\n if (!bytes || bytes.length === 0) return null;\n const batch = deserializeBatch(bytes);\n return batch.numRows === 0 && batch.schema.fields.length === 0 ? null : batch;\n}\n\nfunction serializeBatchWithSchema(batch: VgiBatch): Uint8Array {\n // serializeBatch emits a full IPC stream (schema + batch + EOS) — the C++\n // aggregate_finalize reader expects exactly that.\n return serializeBatch(batch);\n}\n",
|
|
53
53
|
"// Copyright 2025, 2026 Query Farm LLC - https://query.farm\n// Unary handlers for the table_buffering (Sink+Source) lifecycle:\n// table_buffering_process — sink one input batch, return state_id\n// table_buffering_combine — group/merge state_ids -> finalize_state_ids\n// table_buffering_destructor — best-effort end-of-query storage cleanup\n//\n// These RPCs carry no init context, so each cold-loads the init metadata that\n// the TABLE_BUFFERING init phase persisted to FunctionStorage (keyed by\n// execution_id), then rebuilds the user-facing TableBufferingParams. Mirrors\n// vgi-python's worker._load_table_buffering_params.\n\nimport { deserializeBatch } from \"../../arrow/index.js\";\nimport { Protocol } from \"@query-farm/vgi-rpc\";\nimport type { FunctionRegistry } from \"../../functions/registry.js\";\nimport { toUint8Array } from \"../../util/bytes.js\";\nimport { batchToScalarDict, batchToSecretDict } from \"../../util/arrow/index.js\";\nimport {\n TableBufferingCombineParamsSchema,\n TableBufferingCombineResultSchema,\n TableBufferingDestructorParamsSchema,\n TableBufferingDestructorResultSchema,\n TableBufferingProcessParamsSchema,\n TableBufferingProcessResultSchema,\n} from \"../../generated/vgi-protocol-schemas.js\";\nimport {\n REQUEST_PARAMS_SCHEMA,\n RESULT_BINARY_SCHEMA,\n unwrapRequest,\n wrapResult,\n} from \"./shared.js\";\nimport { openAttach } from \"./catalog/shared.js\";\nimport {\n BoundStorage,\n storage as defaultStorage,\n FrameworkNS,\n} from \"../../functions/storage.js\";\nimport {\n decodeBufferingInit,\n type TableBufferingVgiFunction,\n type TableBufferingParams,\n} from \"../../functions/table-buffering.js\";\nimport { Arguments } from \"../../arguments/arguments.js\";\n\nexport function registerTableBufferingMethods(\n protocol: Protocol,\n registry: FunctionRegistry,\n signingKey?: Uint8Array,\n): void {\n // `schemaPath` scopes resolution to the schema the caller named. The\n // buffering process/combine RPCs carry it (protocol 1.2.0), so a name\n // declared in two schemas resolves to the one the request names rather than\n // colliding on the flat by-name index. `null` keeps the cross-schema lookup.\n function resolveBuffering(\n name: string,\n schemaPath?: string[] | null,\n ): TableBufferingVgiFunction {\n const func = registry.get(name, {\n arguments: new Arguments(),\n inputSchema: null,\n isScalar: false,\n schemaPath: schemaPath ?? null,\n }) as any;\n if (!func || func.kind !== \"table_buffering\") {\n throw new Error(`Function '${name}' is not a table_buffering function`);\n }\n return func as TableBufferingVgiFunction;\n }\n\n // Cold-load init metadata + build TableBufferingParams from a unary request.\n async function loadParams(\n inner: Record<string, any>,\n clientLog: (level: string, message: string) => void,\n ctx: any,\n ): Promise<{ func: TableBufferingVgiFunction; params: TableBufferingParams<any> }> {\n const functionName: string = inner.function_name;\n const executionId = toUint8Array(inner.execution_id);\n // Unseal attach_opaque_data so combine()/process() see the SAME plaintext,\n // attach-scoped bytes that bind()/init() and plain table functions see.\n // Without this, on HTTP transport the buffering RPCs would scope persistent\n // BoundStorage by the *sealed* envelope while reads scope by the plaintext,\n // so attach-scoped state written in combine() is invisible to later reads.\n const rawAttach = inner.attach_opaque_data ? toUint8Array(inner.attach_opaque_data) : null;\n const attach =\n rawAttach && rawAttach.length > 0\n ? await openAttach(rawAttach, ctx?.auth, signingKey)\n : rawAttach;\n const transactionId = inner.transaction_id ? toUint8Array(inner.transaction_id) : null;\n\n const func = resolveBuffering(functionName, inner.schema_path ?? null);\n const bound = new BoundStorage(defaultStorage, executionId);\n const payload = await bound.stateGet(\n FrameworkNS.BUFFERING_INIT,\n BoundStorage.packIntKey(-1),\n );\n if (payload == null) {\n throw new Error(\n `table_buffering: unknown execution_id ${hex(executionId)} ` +\n `(init never ran or destructor already fired)`,\n );\n }\n const { request } = decodeBufferingInit(payload);\n const args = func.bufferingExtractArgs(request.bind_call);\n const settings = batchToScalarDict(request.bind_call.settings);\n const secrets = batchToSecretDict(request.bind_call.secrets);\n\n const params: TableBufferingParams<any> = {\n args,\n initCall: request,\n outputSchema: request.output_schema,\n settings,\n secrets,\n storage: bound,\n executionId,\n attachId: attach ?? new Uint8Array(0),\n transactionId,\n function_name: functionName,\n batchIndex: null,\n clientLog,\n };\n return { func, params };\n }\n\n // ------------------------------------------------------------------------\n // table_buffering_process — sink one batch, return state_id (unary)\n // ------------------------------------------------------------------------\n protocol.unary(\"table_buffering_process\", {\n params: TableBufferingProcessParamsSchema,\n result: RESULT_BINARY_SCHEMA,\n handler: async (rpcParams, ctx?: any) => {\n const inner = unwrapRequest(rpcParams.request);\n const clientLog = makeClientLog(ctx);\n const { func, params } = await loadParams(inner, clientLog, ctx);\n if (inner.batch_index != null) {\n params.batchIndex = Number(inner.batch_index);\n }\n const batch = deserializeBatch(toUint8Array(inner.input_batch));\n const stateId = await func.bufferingConfig.process(batch, params);\n if (!(stateId instanceof Uint8Array)) {\n throw new Error(\n `${func.meta.name}.process() must return Uint8Array (the opaque state_id)`,\n );\n }\n return wrapResult({ state_id: stateId }, TableBufferingProcessResultSchema);\n },\n });\n\n // ------------------------------------------------------------------------\n // table_buffering_combine — group/merge state_ids (unary)\n // ------------------------------------------------------------------------\n protocol.unary(\"table_buffering_combine\", {\n params: TableBufferingCombineParamsSchema,\n result: RESULT_BINARY_SCHEMA,\n handler: async (rpcParams, ctx?: any) => {\n const inner = unwrapRequest(rpcParams.request);\n const clientLog = makeClientLog(ctx);\n const { func, params } = await loadParams(inner, clientLog, ctx);\n const stateIds = decodeBytesList(inner.state_ids);\n const finalizeStateIds = await func.bufferingConfig.combine(stateIds, params);\n const out = finalizeStateIds.map((fid, i) => {\n if (!(fid instanceof Uint8Array)) {\n throw new Error(\n `${func.meta.name}.combine() returned non-Uint8Array finalize_state_id at index ${i}`,\n );\n }\n return fid;\n });\n return wrapResult(\n { finalize_state_ids: out },\n TableBufferingCombineResultSchema,\n );\n },\n });\n\n // ------------------------------------------------------------------------\n // table_buffering_destructor — best-effort end-of-query cleanup (unary)\n // ------------------------------------------------------------------------\n protocol.unary(\"table_buffering_destructor\", {\n params: TableBufferingDestructorParamsSchema,\n result: RESULT_BINARY_SCHEMA,\n handler: async (rpcParams) => {\n try {\n const inner = unwrapRequest(rpcParams.request);\n const executionId = toUint8Array(inner.execution_id);\n const bound = new BoundStorage(defaultStorage, executionId);\n await bound.executionClear();\n } catch {\n // Teardown path — swallow; the entry is wiped when the worker exits.\n }\n return wrapResult({}, TableBufferingDestructorResultSchema);\n },\n });\n}\n\nfunction makeClientLog(ctx: any): (level: string, message: string) => void {\n if (ctx && typeof ctx.clientLog === \"function\") {\n return (level: string, message: string) => ctx.clientLog(level, message);\n }\n return () => {};\n}\n\nfunction decodeBytesList(raw: any): Uint8Array[] {\n if (raw == null) return [];\n const out: Uint8Array[] = [];\n const iter: Iterable<any> = Array.isArray(raw)\n ? raw\n : typeof raw[Symbol.iterator] === \"function\"\n ? raw\n : [];\n for (const entry of iter) {\n if (entry == null) continue;\n out.push(toUint8Array(entry));\n }\n return out;\n}\n\nfunction hex(buf: Uint8Array): string {\n return Array.from(buf, (b) => b.toString(16).padStart(2, \"0\")).join(\"\");\n}\n",
|
|
54
54
|
"// Copyright 2025, 2026 Query Farm LLC - https://query.farm\n// Table sink+source (\"table_buffering\") function implementation.\n//\n// Three phases, mirroring the C++ PhysicalVgiTableBuffering Sink+Source\n// operator:\n// 1. Sink — process(batch, params) -> state_id (bytes), one per input\n// batch. Driven by the unary `table_buffering_process` RPC.\n// 2. Combine — combine(state_ids, params) -> finalize_state_ids (bytes[]),\n// once on the coordinator. Driven by `table_buffering_combine`.\n// 3. Source — finalize(params, fid, state, out) producer-mode streaming,\n// one batch per tick. Driven by an init() stream with\n// phase=TABLE_BUFFERING_FINALIZE.\n//\n// CROSS-PROCESS INVARIANT: state written in process() that finalize() reads\n// MUST live in cross-process storage scoped by execution_id (BoundStorage).\n// The Source phase may route a finalize_state_id to a worker process that\n// did NOT run the corresponding process() calls.\n\nimport {\n type VgiSchema,\n schema,\n type VgiDataType,\n type VgiBatch,\n nullType,\n} from \"../arrow/index.js\";\nimport type { OutputCollector } from \"@query-farm/vgi-rpc\";\nimport { DEFAULT_MAX_WORKERS, TableInOutPhase } from \"../types.js\";\nimport type {\n BindRequest,\n BindResponse,\n InitRequest,\n GlobalInitResponse,\n TableFunctionCardinalityRequest,\n} from \"../protocol/types.js\";\nimport type { TableCardinality } from \"../types.js\";\nimport type {\n VgiFunction,\n FunctionMeta,\n StreamHandlers,\n FunctionExample,\n} from \"./types.js\";\nimport type { ArgumentConstraints, ArgumentSpec } from \"../arguments/argument-spec.js\";\nimport {\n assertArrowType,\n assertArrowTypes,\n constraintSpecFields,\n validateConstConstraints,\n} from \"../arguments/argument-spec.js\";\nimport { batchToScalarDict, batchToSecretDict, projectSchema } from \"../util/arrow/index.js\";\nimport { batchFromColumns, readCanonicalValue } from \"../arrow/index.js\";\nimport { codecFor } from \"../arrow/codec/registry.js\";\nimport {\n deserializeFilters,\n FilteringOutputCollector,\n type PushdownFilters,\n} from \"../filter-pushdown/index.js\";\nimport { FunctionStability } from \"../types.js\";\nimport { BoundStorage, storage as defaultStorage, FrameworkNS } from \"./storage.js\";\nimport {\n serializeInitRequest,\n deserializeInitRequest,\n serializeGlobalInitResponse,\n} from \"../protocol/serialize.js\";\nimport { serializeBatch, deserializeBatch } from \"../util/arrow/index.js\";\n\n// ============================================================================\n// Parameter bundles\n// ============================================================================\n\nexport interface TableBufferingBindParams<TArgs = Record<string, any>> {\n args: TArgs;\n bindCall: BindRequest;\n settings: Record<string, any>;\n secrets: Record<string, Record<string, any>>;\n /** True on the second bind pass, after the connector resolved the secrets\n * requested via the onBind `lookupSecret*` return fields on the first pass. */\n resolvedSecretsProvided: boolean;\n /** Resolved names aligned with the complete logical argument order. */\n argumentNames?: (string | null)[] | null;\n}\n\nexport interface TableBufferingParams<TArgs = Record<string, any>> {\n args: TArgs;\n initCall: InitRequest;\n outputSchema: VgiSchema;\n settings: Record<string, any>;\n secrets: Record<string, Record<string, any>>;\n /** Shared cross-process storage scoped by execution_id. */\n storage: BoundStorage;\n /** Stable across coordinator + secondary workers for one DuckDB execution. */\n executionId: Uint8Array;\n /** Catalog attach identity (plaintext bytes). */\n attachId: Uint8Array;\n /** Hex/raw VGI transaction id, or null. */\n transactionId: Uint8Array | null;\n function_name: string;\n /** Per-chunk batch_index when requiresInputBatchIndex=true; else null. */\n batchIndex: number | null;\n /** In-band log sink for the unary process()/combine() RPCs. */\n clientLog: (level: string, message: string) => void;\n}\n\n// ============================================================================\n// Functional API config\n// ============================================================================\n\nexport interface TableBufferingConfig<\n TArgs = Record<string, any>,\n TState = any,\n> {\n name: string;\n description?: string;\n args?: Record<string, VgiDataType>;\n namedArgs?: Record<string, VgiDataType>;\n argDefaults?: Record<string, any>;\n /** Authoritative typed defaults: one row, defaulted parameters only. */\n parameterDefaultValues?: VgiBatch | null;\n /** Argument descriptions keyed by argument name. */\n argDocs?: Record<string, string>;\n /** Discovery constraints, also enforced for scalar arguments at bind. */\n argConstraints?: Record<string, ArgumentConstraints>;\n /** Bind: default passes through input schema. May be async. Return non-empty\n * `lookupSecret*` lists on the first pass to request DuckDB secrets; the\n * connector then re-binds with `resolvedSecretsProvided` and the resolved\n * values are available in the process/finalize params' `secrets`. */\n onBind?: (params: TableBufferingBindParams<TArgs>) =>\n | {\n outputSchema: VgiSchema;\n opaqueData?: Uint8Array;\n lookupSecretTypes?: string[];\n lookupScopes?: string[];\n lookupNames?: string[];\n }\n | Promise<{\n outputSchema: VgiSchema;\n opaqueData?: Uint8Array;\n lookupSecretTypes?: string[];\n lookupScopes?: string[];\n lookupNames?: string[];\n }>;\n /** Sink: ingest one batch, return an opaque state_id. */\n process: (\n batch: VgiBatch,\n params: TableBufferingParams<TArgs>,\n ) => Uint8Array | Promise<Uint8Array>;\n /** Combine: group/merge state_ids, return finalize_state_ids. */\n combine: (\n stateIds: Uint8Array[],\n params: TableBufferingParams<TArgs>,\n ) => Uint8Array[] | Promise<Uint8Array[]>;\n /** Build the initial per-tick finalize state for a finalize_state_id. */\n initialFinalizeState?: (\n finalizeStateId: Uint8Array,\n params: TableBufferingParams<TArgs>,\n ) => TState | Promise<TState>;\n /** Source tick: emit one batch via out.emit / signal EOS via out.finish. */\n finalize: (\n params: TableBufferingParams<TArgs>,\n finalizeStateId: Uint8Array,\n state: TState,\n out: OutputCollector,\n ) => void | Promise<void>;\n cardinality?: (\n params: TableBufferingBindParams<TArgs>,\n ) => TableCardinality | Promise<TableCardinality>;\n // Metadata\n projectionPushdown?: boolean;\n filterPushdown?: boolean;\n autoApplyFilters?: boolean;\n /** Force ParallelSink=false in the C++ operator (single-thread ingest). */\n sinkOrderDependent?: boolean;\n /** Force serial Source drain in finalize_queue order. */\n sourceOrderDependent?: boolean;\n /** Thread DuckDB's per-chunk batch_index into every process() call. */\n requiresInputBatchIndex?: boolean;\n stability?: FunctionStability;\n examples?: FunctionExample[];\n categories?: string[];\n tags?: Record<string, string>;\n maxWorkers?: number;\n requiredSettings?: string[];\n requiredSecrets?: string[];\n}\n\n/**\n * A table_buffering VgiFunction also carries its callback config so the\n * protocol's unary handlers (process/combine/destructor) can reach it.\n */\nexport interface TableBufferingVgiFunction extends VgiFunction {\n bufferingConfig: TableBufferingConfig<any, any>;\n bufferingExtractArgs: (request: BindRequest) => any;\n}\n\nexport function defineTableBufferingFunction<\n TArgs = Record<string, any>,\n TState = any,\n>(config: TableBufferingConfig<TArgs, TState>): TableBufferingVgiFunction {\n assertArrowTypes((config as any).args, `defineTableBufferingFunction(\"${config.name}\"): args`);\n\n if (config.sinkOrderDependent && config.requiresInputBatchIndex) {\n throw new Error(\n `${config.name}: sinkOrderDependent and requiresInputBatchIndex are ` +\n `mutually exclusive — single-thread sink already orders input.`,\n );\n }\n\n const specs: ArgumentSpec[] = [];\n let posIdx = 0;\n if (config.args) {\n for (const [name, type] of Object.entries(config.args)) {\n specs.push({\n name,\n position: posIdx++,\n arrowType: type,\n doc: config.argDocs?.[name],\n ...constraintSpecFields(\n config.argConstraints?.[name],\n config.argDefaults?.[name],\n ),\n });\n }\n }\n specs.push({\n name: \"data\",\n position: posIdx++,\n arrowType: nullType(),\n isTableInput: true,\n });\n if (config.namedArgs) {\n for (const [name, type] of Object.entries(config.namedArgs)) {\n specs.push({\n name,\n position: name,\n arrowType: type,\n doc: config.argDocs?.[name],\n ...constraintSpecFields(\n config.argConstraints?.[name],\n config.argDefaults?.[name],\n ),\n });\n }\n }\n\n const meta: FunctionMeta = {\n name: config.name,\n description: config.description,\n stability: config.stability,\n projectionPushdown: config.projectionPushdown,\n filterPushdown: config.filterPushdown,\n autoApplyFilters: config.autoApplyFilters,\n examples: config.examples,\n categories: config.categories,\n tags: config.tags,\n parameterDefaultValues: config.parameterDefaultValues,\n maxWorkers: config.maxWorkers,\n requiredSettings: config.requiredSettings,\n requiredSecrets: config.requiredSecrets,\n // The Source phase always runs (it's how output is produced), so a\n // table_buffering function always advertises has_finalize.\n hasFinalize: true,\n sinkOrderDependent: config.sinkOrderDependent,\n sourceOrderDependent: config.sourceOrderDependent,\n requiresInputBatchIndex: config.requiresInputBatchIndex,\n };\n\n function extractArgs(request: BindRequest): TArgs {\n const args: Record<string, any> = {};\n for (const spec of specs) {\n if (spec.isTableInput) continue;\n const defaultVal =\n config.argDefaults?.[spec.name] !== undefined\n ? config.argDefaults[spec.name]\n : undefined;\n let val = request.arguments.get(spec.position, defaultVal);\n if (typeof val === \"bigint\") val = Number(val);\n const constraints = config.argConstraints?.[spec.name];\n if (constraints) validateConstConstraints(spec.name, constraints, val);\n args[spec.name] = val;\n }\n return args as TArgs;\n }\n\n const func: TableBufferingVgiFunction = {\n kind: \"table_buffering\" as any,\n meta,\n argumentSpecs: specs,\n bufferingConfig: config as TableBufferingConfig<any, any>,\n bufferingExtractArgs: extractArgs as any,\n\n async bind(request: BindRequest): Promise<BindResponse> {\n if (config.onBind) {\n const args = extractArgs(request);\n const settings = batchToScalarDict(request.settings);\n const secrets = batchToSecretDict(request.secrets);\n const result = await config.onBind({\n args,\n bindCall: request,\n settings,\n secrets,\n resolvedSecretsProvided: request.resolved_secrets_provided ?? false,\n argumentNames: request.argument_names ?? null,\n });\n return {\n output_schema: result.outputSchema,\n opaque_data: result.opaqueData ?? null,\n lookup_secret_types: result.lookupSecretTypes,\n lookup_scopes: result.lookupScopes,\n lookup_names: result.lookupNames,\n };\n }\n return {\n output_schema: request.input_schema ?? schema([]),\n opaque_data: null,\n };\n },\n\n async globalInit(request: InitRequest): Promise<GlobalInitResponse> {\n // Mint a stable execution_id for the primary; secondary inits reuse it.\n let executionId: Uint8Array;\n if (request.execution_id) {\n executionId = request.execution_id;\n } else {\n executionId = new Uint8Array(16);\n crypto.getRandomValues(executionId);\n }\n const response: GlobalInitResponse = {\n max_workers: config.maxWorkers ?? DEFAULT_MAX_WORKERS,\n execution_id: executionId,\n opaque_data: null,\n };\n\n // On the sink-side init, persist init metadata so any pool worker can\n // cold-load it when serving subsequent process()/combine() unary RPCs\n // (which carry no init context themselves). Mirrors Python's\n // FrameworkNS.BUFFERING_INIT state_put.\n if (request.phase === TableInOutPhase.TABLE_BUFFERING) {\n const bound = new BoundStorage(defaultStorage, executionId);\n await bound.statePut(\n FrameworkNS.BUFFERING_INIT,\n BoundStorage.packIntKey(-1),\n encodeBufferingInit(request, response),\n );\n }\n return response;\n },\n\n createStreamHandlers(\n request: InitRequest,\n response: GlobalInitResponse,\n ): StreamHandlers {\n const args = extractArgs(request.bind_call);\n const settings = batchToScalarDict(request.bind_call.settings);\n const secrets = batchToSecretDict(request.bind_call.secrets);\n\n const projIds = request.projection_ids && meta.projectionPushdown\n ? request.projection_ids\n : null;\n const outputSchema = projIds\n ? projectSchema(projIds, request.output_schema)\n : request.output_schema;\n\n const pushdownFilters = request.pushdown_filters\n ? deserializeFilters(request.pushdown_filters, {\n outputSchema: request.output_schema,\n joinKeyBatches: request.join_keys,\n extensionFunctions: meta.additionalFilterFunctions,\n runtimeAlgorithms: meta.runtimeFilterAlgorithms,\n evaluationContexts: meta.filterEvaluationContexts,\n })\n : undefined;\n\n const bound = new BoundStorage(\n defaultStorage,\n response.execution_id ?? new Uint8Array(16),\n );\n\n const params: TableBufferingParams<TArgs> = {\n args,\n initCall: request,\n outputSchema,\n settings,\n secrets,\n storage: bound,\n executionId: response.execution_id,\n attachId: request.bind_call.attach_opaque_data ?? new Uint8Array(0),\n transactionId: request.bind_call.transaction_opaque_data ?? null,\n function_name: request.bind_call.function_name,\n batchIndex: null,\n clientLog: () => {},\n };\n\n if (request.phase === TableInOutPhase.TABLE_BUFFERING_FINALIZE) {\n // SOURCE phase: producer-mode streaming, one batch per tick.\n const finalizeStateId = request.finalize_state_id ?? new Uint8Array(0);\n return {\n outputSchema,\n producerInit: () => ({ state: { __tbf_uninit: true } as any }),\n producerFn: async (\n pState: { state: any },\n out: OutputCollector,\n ) => {\n // Lazily build the user's per-tick state on the first tick.\n // pState.state round-trips through the HTTP exchange-state token\n // between ticks; `__tbf_uninit` marks \"not yet initialized\".\n let userState: any = pState.state;\n if (userState && userState.__tbf_uninit) {\n userState = config.initialFinalizeState\n ? await config.initialFinalizeState(finalizeStateId, params)\n : null;\n pState.state = userState ?? null;\n } else {\n userState = pState.state;\n }\n\n // Narrow each emitted (full-width) buffered batch to the projected\n // output schema, then auto-apply pushdown filters. Mirrors the\n // streaming TableFunctionGenerator path: project first, filter\n // second.\n let wrappedOut: OutputCollector = projIds\n ? makeProjectingCollector(out, outputSchema)\n : out;\n if (config.autoApplyFilters && pushdownFilters) {\n wrappedOut = new FilteringOutputCollector(wrappedOut, pushdownFilters) as unknown as OutputCollector;\n }\n\n await config.finalize(params, finalizeStateId, userState, wrappedOut);\n\n // Persist the (possibly mutated) user state for the next tick.\n pState.state = userState ?? null;\n },\n };\n }\n\n // SINK-init phase (TABLE_BUFFERING): the C++ side opens the init stream\n // in exchange mode, immediately sends EOS, and drains output to EOS.\n // We produce nothing — the real sink traffic is the separate unary\n // table_buffering_process / _combine RPCs.\n return {\n outputSchema,\n inputSchema: request.bind_call.input_schema ?? undefined,\n exchangeInit: () => ({ state: null }),\n exchangeFn: async (\n _state: any,\n _input: VgiBatch,\n _out: OutputCollector,\n ) => {\n // No-op: no per-batch output on the sink-init stream.\n },\n };\n },\n };\n\n if (config.cardinality) {\n func.cardinality = async (request: TableFunctionCardinalityRequest) => {\n const args = extractArgs(request.bind_call);\n const settings = batchToScalarDict(request.bind_call.settings);\n const secrets = batchToSecretDict(request.bind_call.secrets);\n return config.cardinality!({\n args,\n bindCall: request.bind_call,\n settings,\n secrets,\n resolvedSecretsProvided: request.bind_call.resolved_secrets_provided ?? false,\n });\n };\n }\n\n return func;\n}\n\n// Wrap an OutputCollector so each emitted batch is projected-by-name to the\n// target (narrowed) output schema before forwarding. Mirrors table.ts's\n// makeProjectingCollector — buffered batches are stored full-width, so the\n// Source phase narrows them per-tick to the projected columns.\nfunction makeProjectingCollector(\n inner: OutputCollector,\n targetSchema: VgiSchema,\n): OutputCollector {\n function alreadyMatches(batch: VgiBatch): boolean {\n if (batch.schema.fields.length !== targetSchema.fields.length) return false;\n for (let i = 0; i < targetSchema.fields.length; i++) {\n if (batch.schema.fields[i].name !== targetSchema.fields[i].name) return false;\n }\n return true;\n }\n function project(batch: VgiBatch): VgiBatch {\n if (alreadyMatches(batch)) return batch;\n const cols: Record<string, any[]> = {};\n for (const f of targetSchema.fields) {\n const src = batch.getChild(f.name);\n if (src) {\n // Canonical read -> rich so the rebuild goes through the codec path\n // (backend-agnostic, lossless) rather than the raw `.get(i)`.\n const type = f.type as VgiDataType;\n const codec = codecFor(type);\n const arr: any[] = [];\n for (let i = 0; i < batch.numRows; i++) {\n arr.push(codec.canonicalToRich(readCanonicalValue(type, src, i)));\n }\n cols[f.name] = arr;\n } else {\n cols[f.name] = new Array(batch.numRows).fill(null);\n }\n }\n return batchFromColumns(cols, targetSchema);\n }\n return new Proxy(inner, {\n get(target, prop, receiver) {\n if (prop === \"emit\") {\n return function (\n this: OutputCollector,\n batchOrColumns: VgiBatch | Record<string, any[]>,\n metadata?: Map<string, string>,\n ) {\n if (batchOrColumns && typeof (batchOrColumns as any).getChild === \"function\") {\n return (target as any).emit(project(batchOrColumns as VgiBatch), metadata);\n }\n return (target as any).emit(batchOrColumns, metadata);\n };\n }\n const v = Reflect.get(target, prop, receiver);\n return typeof v === \"function\" ? v.bind(target) : v;\n },\n });\n}\n\n// ============================================================================\n// Init-metadata persistence (mirrors Python _encode/_decode_table_buffering_init)\n// ============================================================================\n\nexport function encodeBufferingInit(\n request: InitRequest,\n response: GlobalInitResponse,\n): Uint8Array {\n // Serialize the InitRequest batch and the GlobalInitResponse so any pool\n // worker can cold-rebuild params when serving a process()/combine() RPC.\n const initBatch = serializeInitRequest(request);\n const initBytes = serializeBatch(initBatch);\n const respBatch = serializeGlobalInitResponse(response);\n // GlobalInitResponse serializer returns a plain dict; rebuild a 1-row batch.\n // Simpler: re-serialize as a length-prefixed pair of byte blobs.\n const respBytes = encodeGlobalInitResponse(respBatch);\n return concatLenPrefixed([initBytes, respBytes]);\n}\n\nexport function decodeBufferingInit(\n payload: Uint8Array,\n): { request: InitRequest; response: GlobalInitResponse } {\n const [initBytes, respBytes] = splitLenPrefixed(payload, 2);\n const initBatch = deserializeBatch(initBytes);\n const initDict = batchToScalarDict(initBatch);\n const request = deserializeInitRequest(initDict);\n const response = decodeGlobalInitResponse(respBytes);\n return { request, response };\n}\n\nfunction encodeGlobalInitResponse(dict: Record<string, any>): Uint8Array {\n // Encode { execution_id, opaque_data, max_workers } as length-prefixed blobs.\n const eid: Uint8Array = dict.execution_id ?? new Uint8Array(0);\n const opaque: Uint8Array | null = dict.opaque_data ?? null;\n const maxWorkers = Number(dict.max_workers ?? 1);\n const mwBuf = new Uint8Array(8);\n new DataView(mwBuf.buffer).setBigInt64(0, BigInt(maxWorkers), true);\n return concatLenPrefixed([eid, opaque ?? new Uint8Array(0), mwBuf]);\n}\n\nfunction decodeGlobalInitResponse(bytes: Uint8Array): GlobalInitResponse {\n const [eid, opaque, mwBuf] = splitLenPrefixed(bytes, 3);\n const maxWorkers = Number(new DataView(mwBuf.buffer, mwBuf.byteOffset, 8).getBigInt64(0, true));\n return {\n execution_id: eid,\n opaque_data: opaque.length > 0 ? opaque : null,\n max_workers: maxWorkers,\n };\n}\n\nfunction concatLenPrefixed(parts: Uint8Array[]): Uint8Array {\n let total = 0;\n for (const p of parts) total += 4 + p.length;\n const out = new Uint8Array(total);\n const dv = new DataView(out.buffer);\n let off = 0;\n for (const p of parts) {\n dv.setUint32(off, p.length, true);\n off += 4;\n out.set(p, off);\n off += p.length;\n }\n return out;\n}\n\nfunction splitLenPrefixed(buf: Uint8Array, count: number): Uint8Array[] {\n const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);\n const parts: Uint8Array[] = [];\n let off = 0;\n for (let i = 0; i < count; i++) {\n const len = dv.getUint32(off, true);\n off += 4;\n parts.push(buf.subarray(off, off + len));\n off += len;\n }\n return parts;\n}\n",
|
|
55
|
-
"// Copyright 2025, 2026 Query Farm LLC - https://query.farm\n// SPDX-License-Identifier: LicenseRef-QueryFarm-Source-Available-1.0\n\n/**\n * Generic ArrowSerializableDataclass (ASD) codec.\n *\n * Bridges between typed TS objects (snake_case, matching vgi-python dataclass\n * field names) and Arrow IPC bytes (single-row batch with the dataclass'\n * ARROW_SCHEMA). Used by the generated vgi-client.ts encode<X>/decode<X>\n * wrappers to make the typed client interfaces honest at the wire boundary.\n *\n * Encoding delegates to the facade's `batchFromRows` (which handles the\n * complex types — Decimal/BigInt/List/Map/Struct/Dictionary — uniformly\n * across both arrow-js and flechette backends). Decoding pulls each field's\n * value with `batchToScalarDict`, then runs `normalizeValue` to convert\n * Arrow representations (MapRow, BigInt, etc.) back into plain JS shapes.\n */\n\nimport {\n type VgiSchema,\n type VgiField,\n type VgiDataType,\n isBinary,\n isBool,\n isDictionary,\n isInt,\n isList,\n isMap,\n isStruct,\n batchFromRows,\n serializeBatch,\n deserializeBatch,\n} from \"../arrow/index.js\";\n\n/**\n * Encode a typed object as a single-row Arrow IPC stream using `schema`.\n * Matches Python's `ArrowSerializableDataclass.serialize_to_bytes()`.\n */\nexport function encodeASD(\n schema: VgiSchema,\n obj: Record<string, any>,\n): Uint8Array {\n return serializeBatch(batchFromRows([sortMapsForSchema(schema, obj)], schema));\n}\n\n// --------------------------------------------------------------------------- //\n// Encode side: deterministic map columns\n// --------------------------------------------------------------------------- //\n//\n// A map value arrives as a JS object (or Map / pair list), whose iteration\n// order is insertion order -- except that integer-like keys (\"1\", \"2\") always\n// come first, ascending. Two builds of the same record could therefore encode\n// differently (and a catalog_contents content-hash etag would change with no\n// catalog change). Every map is written in sorted key order instead, matching\n// the Go / Java / C# / Rust SDKs: keys compared by Unicode code point (the\n// same order as comparing their UTF-8 bytes), not by UTF-16 code unit.\n\n/** Compare two strings by Unicode code point (= UTF-8 byte order). */\nexport function compareCodePoints(a: string, b: string): number {\n const n = Math.min(a.length, b.length);\n for (let i = 0; i < n; i++) {\n const x = a.charCodeAt(i);\n const y = b.charCodeAt(i);\n if (x === y) continue;\n // A surrogate (a code point above U+FFFF) sorts after every BMP code unit,\n // though as a code unit (0xD800-0xDFFF) it sorts below U+E000-U+FFFF.\n const xs = x >= 0xd800 && x <= 0xdfff;\n const ys = y >= 0xd800 && y <= 0xdfff;\n if (xs !== ys) return xs ? 1 : -1;\n return x - y;\n }\n return a.length - b.length;\n}\n\nfunction mapPairs(raw: any): Array<[unknown, unknown]> {\n if (Array.isArray(raw)) return raw.map((e: any) => (Array.isArray(e) ? [e[0], e[1]] : [e?.key, e?.value]));\n if (raw instanceof Map) return Array.from(raw.entries());\n if (typeof raw[Symbol.iterator] === \"function\") return Array.from(raw as Iterable<[unknown, unknown]>);\n return Object.entries(raw);\n}\n\nfunction compareKeys(a: unknown, b: unknown): number {\n if (typeof a === \"string\" && typeof b === \"string\") return compareCodePoints(a, b);\n if ((typeof a === \"number\" || typeof a === \"bigint\") && (typeof b === \"number\" || typeof b === \"bigint\")) {\n return a < b ? -1 : a > b ? 1 : 0;\n }\n return compareCodePoints(String(a), String(b));\n}\n\n/** Whether a type contains a map anywhere (memoized per type object). */\nconst HAS_MAP = new WeakMap<object, boolean>();\nfunction typeHasMap(type: VgiDataType): boolean {\n const cached = HAS_MAP.get(type as object);\n if (cached !== undefined) return cached;\n let has = isMap(type);\n if (!has && (isList(type) || isStruct(type))) {\n has = ((type as any).children as VgiField[]).some((c) => typeHasMap(c.type));\n }\n HAS_MAP.set(type as object, has);\n return has;\n}\n\nfunction sortMapsForType(type: VgiDataType, value: any): any {\n if (value == null || !typeHasMap(type)) return value;\n if (isMap(type)) {\n const entries = (type as any).children[0].type as VgiDataType;\n const valueType = ((entries as any).children as VgiField[])[1].type;\n return mapPairs(value)\n .map(([k, v]): [unknown, unknown] => [k, sortMapsForType(valueType, v)])\n .sort((x, y) => compareKeys(x[0], y[0]));\n }\n if (isList(type)) {\n const child = (type as any).children[0].type as VgiDataType;\n return Array.from(value as Iterable<unknown>, (v) => sortMapsForType(child, v));\n }\n if (isStruct(type)) {\n const out: Record<string, any> = { ...value };\n for (const cf of (type as any).children as VgiField[]) out[cf.name] = sortMapsForType(cf.type, value[cf.name]);\n return out;\n }\n return value;\n}\n\n/**\n * `obj` with every map-typed field (at any depth) as a key-sorted pair list.\n * Returns `obj` itself when the schema has no map column. Never mutates `obj`\n * (catalog items may be frozen and shared).\n */\nexport function sortMapsForSchema(schema: VgiSchema, obj: Record<string, any>): Record<string, any> {\n let out: Record<string, any> | null = null;\n for (const f of schema.fields) {\n if (!typeHasMap(f.type)) continue;\n const v = obj[f.name];\n if (v == null) continue;\n out ??= { ...obj };\n out[f.name] = sortMapsForType(f.type, v);\n }\n return out ?? obj;\n}\n\n/**\n * Decode a single-row Arrow IPC stream into a typed object using `schema`.\n * Matches Python's `ArrowSerializableDataclass.deserialize_from_bytes()`.\n */\nexport function decodeASD<T>(\n schema: VgiSchema,\n bytes: Uint8Array,\n): T {\n const batch = deserializeBatch(bytes);\n if (batch.numRows === 0) {\n const names = schema.fields.map((f) => f.name).join(\",\");\n throw new Error(`decodeASD: empty batch (expected 1 row for ${names})`);\n }\n const out: Record<string, any> = {};\n for (const field of schema.fields) {\n const col = batch.getChild(field.name);\n const raw = col ? col.get(0) : null;\n out[field.name] = normalizeValue(raw, field.type);\n }\n return out as T;\n}\n\n// --------------------------------------------------------------------------- //\n// Decode side: Arrow value -> normalized JS value\n// --------------------------------------------------------------------------- //\n\nfunction normalizeValue(raw: any, type: VgiDataType): any {\n if (raw == null) return null;\n\n if (isList(type)) {\n const childType = (type as any).children[0].type as VgiDataType;\n const out: any[] = [];\n for (const item of raw) out.push(normalizeValue(item, childType));\n return out;\n }\n\n if (isMap(type)) {\n const out: Record<string,
|
|
55
|
+
"// Copyright 2025, 2026 Query Farm LLC - https://query.farm\n// SPDX-License-Identifier: LicenseRef-QueryFarm-Source-Available-1.0\n\n/**\n * Generic ArrowSerializableDataclass (ASD) codec.\n *\n * Bridges between typed TS objects (snake_case, matching vgi-python dataclass\n * field names) and Arrow IPC bytes (single-row batch with the dataclass'\n * ARROW_SCHEMA). Used by the generated vgi-client.ts encode<X>/decode<X>\n * wrappers to make the typed client interfaces honest at the wire boundary.\n *\n * Encoding delegates to the facade's `batchFromRows` (which handles the\n * complex types — Decimal/BigInt/List/Map/Struct/Dictionary — uniformly\n * across both arrow-js and flechette backends). Decoding pulls each field's\n * value with `batchToScalarDict`, then runs `normalizeValue` to convert\n * Arrow representations (MapRow, BigInt, etc.) back into plain JS shapes.\n */\n\nimport {\n type VgiSchema,\n type VgiField,\n type VgiDataType,\n isBinary,\n isBool,\n isDictionary,\n isInt,\n isList,\n isMap,\n isStruct,\n batchFromRows,\n serializeBatch,\n deserializeBatch,\n} from \"../arrow/index.js\";\n\n/**\n * Encode a typed object as a single-row Arrow IPC stream using `schema`.\n * Matches Python's `ArrowSerializableDataclass.serialize_to_bytes()`.\n */\nexport function encodeASD(\n schema: VgiSchema,\n obj: Record<string, any>,\n): Uint8Array {\n return serializeBatch(batchFromRows([sortMapsForSchema(schema, obj)], schema));\n}\n\n// --------------------------------------------------------------------------- //\n// Encode side: deterministic map columns\n// --------------------------------------------------------------------------- //\n//\n// A map value arrives as a JS object (or Map / pair list), whose iteration\n// order is insertion order -- except that integer-like keys (\"1\", \"2\") always\n// come first, ascending. Two builds of the same record could therefore encode\n// differently (and a catalog_contents content-hash etag would change with no\n// catalog change). Every map is written in sorted key order instead, matching\n// the Go / Java / C# / Rust SDKs: keys compared by Unicode code point (the\n// same order as comparing their UTF-8 bytes), not by UTF-16 code unit.\n\n/** Compare two strings by Unicode code point (= UTF-8 byte order). */\nexport function compareCodePoints(a: string, b: string): number {\n const n = Math.min(a.length, b.length);\n for (let i = 0; i < n; i++) {\n const x = a.charCodeAt(i);\n const y = b.charCodeAt(i);\n if (x === y) continue;\n // A surrogate (a code point above U+FFFF) sorts after every BMP code unit,\n // though as a code unit (0xD800-0xDFFF) it sorts below U+E000-U+FFFF.\n const xs = x >= 0xd800 && x <= 0xdfff;\n const ys = y >= 0xd800 && y <= 0xdfff;\n if (xs !== ys) return xs ? 1 : -1;\n return x - y;\n }\n return a.length - b.length;\n}\n\nfunction mapPairs(raw: any): Array<[unknown, unknown]> {\n if (Array.isArray(raw)) return raw.map((e: any) => (Array.isArray(e) ? [e[0], e[1]] : [e?.key, e?.value]));\n if (raw instanceof Map) return Array.from(raw.entries());\n if (typeof raw[Symbol.iterator] === \"function\") return Array.from(raw as Iterable<[unknown, unknown]>);\n return Object.entries(raw);\n}\n\nfunction compareKeys(a: unknown, b: unknown): number {\n if (typeof a === \"string\" && typeof b === \"string\") return compareCodePoints(a, b);\n if ((typeof a === \"number\" || typeof a === \"bigint\") && (typeof b === \"number\" || typeof b === \"bigint\")) {\n return a < b ? -1 : a > b ? 1 : 0;\n }\n return compareCodePoints(String(a), String(b));\n}\n\n/** Whether a type contains a map anywhere (memoized per type object). */\nconst HAS_MAP = new WeakMap<object, boolean>();\nfunction typeHasMap(type: VgiDataType): boolean {\n const cached = HAS_MAP.get(type as object);\n if (cached !== undefined) return cached;\n let has = isMap(type);\n if (!has && (isList(type) || isStruct(type))) {\n has = ((type as any).children as VgiField[]).some((c) => typeHasMap(c.type));\n }\n HAS_MAP.set(type as object, has);\n return has;\n}\n\nfunction sortMapsForType(type: VgiDataType, value: any): any {\n if (value == null || !typeHasMap(type)) return value;\n if (isMap(type)) {\n const entries = (type as any).children[0].type as VgiDataType;\n const valueType = ((entries as any).children as VgiField[])[1].type;\n return mapPairs(value)\n .map(([k, v]): [unknown, unknown] => [k, sortMapsForType(valueType, v)])\n .sort((x, y) => compareKeys(x[0], y[0]));\n }\n if (isList(type)) {\n const child = (type as any).children[0].type as VgiDataType;\n return Array.from(value as Iterable<unknown>, (v) => sortMapsForType(child, v));\n }\n if (isStruct(type)) {\n const out: Record<string, any> = { ...value };\n for (const cf of (type as any).children as VgiField[]) out[cf.name] = sortMapsForType(cf.type, value[cf.name]);\n return out;\n }\n return value;\n}\n\n/**\n * `obj` with every map-typed field (at any depth) as a key-sorted pair list.\n * Returns `obj` itself when the schema has no map column. Never mutates `obj`\n * (catalog items may be frozen and shared).\n */\nexport function sortMapsForSchema(schema: VgiSchema, obj: Record<string, any>): Record<string, any> {\n let out: Record<string, any> | null = null;\n for (const f of schema.fields) {\n if (!typeHasMap(f.type)) continue;\n const v = obj[f.name];\n if (v == null) continue;\n out ??= { ...obj };\n out[f.name] = sortMapsForType(f.type, v);\n }\n return out ?? obj;\n}\n\n/**\n * Decode a single-row Arrow IPC stream into a typed object using `schema`.\n * Matches Python's `ArrowSerializableDataclass.deserialize_from_bytes()`.\n */\nexport function decodeASD<T>(\n schema: VgiSchema,\n bytes: Uint8Array,\n): T {\n const batch = deserializeBatch(bytes);\n if (batch.numRows === 0) {\n const names = schema.fields.map((f) => f.name).join(\",\");\n throw new Error(`decodeASD: empty batch (expected 1 row for ${names})`);\n }\n const out: Record<string, any> = {};\n for (const field of schema.fields) {\n const col = batch.getChild(field.name);\n const raw = col ? col.get(0) : null;\n out[field.name] = normalizeValue(raw, field.type);\n }\n return out as T;\n}\n\n// --------------------------------------------------------------------------- //\n// Decode side: Arrow value -> normalized JS value\n// --------------------------------------------------------------------------- //\n\nfunction normalizeValue(raw: any, type: VgiDataType): any {\n if (raw == null) return null;\n\n if (isList(type)) {\n const childType = (type as any).children[0].type as VgiDataType;\n const out: any[] = [];\n for (const item of raw) out.push(normalizeValue(item, childType));\n return out;\n }\n\n if (isMap(type)) {\n // Values are decoded with the map's declared value type, like list items\n // and struct children: a map<utf8, int64> yields numbers (bigint beyond\n // the safe range), not strings, and a null value stays null.\n const entries = (type as any).children[0].type as VgiDataType;\n const valueType = ((entries as any).children as VgiField[])[1].type;\n const out: Record<string, any> = {};\n const put = (k: unknown, v: unknown): void => {\n out[String(k)] = normalizeValue(v, valueType);\n };\n // arrow-js MapRow is iterable of [k,v]; flechette returns [[k,v],...] arrays.\n if (raw[Symbol.iterator]) {\n for (const entry of raw) {\n if (Array.isArray(entry)) {\n put(entry[0], entry[1]);\n } else if (entry && typeof entry === \"object\") {\n put(entry.key ?? entry[0], entry.value ?? entry[1]);\n }\n }\n } else if (typeof raw === \"object\") {\n // Plain-object map (flechette without useMap): Object.entries fallback\n for (const [k, v] of Object.entries(raw)) put(k, v);\n }\n return out;\n }\n\n if (isStruct(type)) {\n const childFields = (type as any).children as VgiField[];\n const out: Record<string, any> = {};\n for (const cf of childFields) {\n out[cf.name] = normalizeValue(raw[cf.name], cf.type);\n }\n return out;\n }\n\n if (isDictionary(type)) {\n return raw == null ? null : String(raw);\n }\n\n if (isBinary(type)) {\n return toUint8Array(raw);\n }\n\n if (isInt(type) && (type as any).bitWidth === 64) {\n // Prefer number when representable; callers that need BigInt can convert.\n if (typeof raw === \"bigint\") {\n if (raw >= BigInt(Number.MIN_SAFE_INTEGER) && raw <= BigInt(Number.MAX_SAFE_INTEGER)) {\n return Number(raw);\n }\n return raw;\n }\n return raw;\n }\n\n if (isBool(type)) {\n return Boolean(raw);\n }\n\n // Utf8, Int8/16/32, Float32/64 — pass through as-is.\n return raw;\n}\n\nfunction toUint8Array(val: any): Uint8Array {\n if (val instanceof Uint8Array) return val;\n if (val instanceof ArrayBuffer) return new Uint8Array(val);\n if (val && val.buffer instanceof ArrayBuffer) {\n return new Uint8Array(val.buffer, val.byteOffset, val.byteLength);\n }\n return new Uint8Array(0);\n}\n",
|
|
56
56
|
"// ============================================================================\n// GENERATED by vgi.codegen.ts_client. DO NOT EDIT BY HAND.\n//\n// Generator: vgi-gen-ts-client v1\n// Content hash: 5cf87195498c\n//\n// To regenerate:\n// uv run --project ~/Development/vgi-python vgi-gen-ts-client \\\n// > ~/Development/vgi-typescript/src/generated/vgi-client.ts\n// ============================================================================\n\nimport { encodeASD, decodeASD } from \"../codec/asd.js\";\nimport {\n CatalogInfoSchema,\n FunctionInfoSchema,\n IndexInfoSchema,\n MacroInfoSchema,\n SchemaInfoSchema,\n TableInfoSchema,\n ViewInfoSchema,\n} from \"./vgi-protocol-schemas.js\";\n\nexport type ArgumentMonotonicity = \"UNKNOWN\" | \"CONSTANT\" | \"NON_DECREASING\" | \"STRICTLY_INCREASING\" | \"NON_INCREASING\" | \"STRICTLY_DECREASING\";\nexport type DistinctDependence = \"DISTINCT_DEPENDENT\" | \"NOT_DISTINCT_DEPENDENT\";\nexport type FunctionStability = \"CONSISTENT\" | \"VOLATILE\" | \"CONSISTENT_WITHIN_QUERY\";\nexport type FunctionType = \"SCALAR\" | \"TABLE\" | \"TABLE_BUFFERING\" | \"AGGREGATE\";\nexport type IndexConstraintType = \"NONE\" | \"UNIQUE\" | \"PRIMARY\";\nexport type InvocationFunctionType = \"AGGREGATE\" | \"SCALAR\" | \"TABLE\" | \"TABLE_BUFFERING\";\nexport type MacroType = \"SCALAR\" | \"TABLE\";\nexport type NullHandling = \"DEFAULT\" | \"SPECIAL\";\nexport type OnConflict = \"ERROR\" | \"IGNORE\" | \"REPLACE\";\nexport type OrderByDirection = \"ASC\" | \"DESC\";\nexport type OrderByNullOrder = \"NULLS_FIRST\" | \"NULLS_LAST\";\nexport type OrderDependence = \"ORDER_DEPENDENT\" | \"NOT_ORDER_DEPENDENT\";\nexport type OrderPreservation = \"PRESERVES_ORDER\" | \"NO_ORDER_GUARANTEE\" | \"FIXED_ORDER\";\nexport type PartitionKind = \"NOT_PARTITIONED\" | \"SINGLE_VALUE_PARTITIONS\" | \"OVERLAPPING_PARTITIONS\" | \"DISJOINT_PARTITIONS\";\nexport type SchemaObjectType = \"TABLE\" | \"VIEW\" | \"SCALAR_FUNCTION\" | \"TABLE_FUNCTION\" | \"AGGREGATE_FUNCTION\" | \"SCALAR_MACRO\" | \"TABLE_MACRO\" | \"INDEX\";\nexport type TableInOutFunctionInitPhase = \"INPUT\" | \"FINALIZE\" | \"TABLE_BUFFERING\" | \"TABLE_BUFFERING_FINALIZE\";\n\nexport interface CatalogDataVersionRelease {\n version: string;\n released_at?: bigint | null;\n summary: string;\n notes_url?: string | null;\n}\n\nexport interface CatalogInfo {\n name: string;\n implementation_version?: string | null;\n data_version_spec?: string | null;\n attach_option_specs: Uint8Array[];\n releases: CatalogDataVersionRelease[];\n source_url?: string | null;\n}\n\nexport interface SchemaInfo {\n comment?: string | null;\n tags: Record<string, string>;\n attach_opaque_data: Uint8Array;\n path: string[];\n estimated_object_count?: Record<string, number> | null;\n}\n\nexport interface TableInfo {\n comment?: string | null;\n tags: Record<string, string>;\n name: string;\n schema_path: string[];\n columns: Uint8Array;\n not_null_constraints: number[];\n unique_constraints: number[][];\n check_constraints: string[];\n primary_key_constraints: number[][];\n foreign_key_constraints: Uint8Array[];\n write_result_modes: Record<string, string>;\n supports_column_statistics: boolean;\n scan_function?: Uint8Array | null;\n insert_function?: Uint8Array | null;\n update_function?: Uint8Array | null;\n delete_function?: Uint8Array | null;\n cardinality_estimate?: number | null;\n cardinality_max?: number | null;\n column_statistics?: Uint8Array | null;\n bind_result?: Uint8Array | null;\n required_filters: string[][];\n}\n\nexport interface ViewInfo {\n comment?: string | null;\n tags: Record<string, string>;\n name: string;\n schema_path: string[];\n definition: string;\n column_comments: Record<string, string>;\n}\n\nexport interface CatalogExample {\n sql: string;\n description: string;\n expected_output?: string | null;\n}\n\nexport interface FilterFunctionCapability {\n namespace: string;\n name: string;\n version: number;\n}\n\nexport interface RuntimeFilterAlgorithmCapability {\n namespace: string;\n name: string;\n version: number;\n}\n\nexport interface EvaluationContextCapability {\n profile: string;\n provider_fingerprint?: string | null;\n}\n\nexport interface SecretLookupEntry {\n secret_type: string;\n scope?: string | null;\n secret_name?: string | null;\n}\n\nexport interface FunctionInfo {\n comment?: string | null;\n tags: Record<string, string>;\n name: string;\n schema_path: string[];\n function_type: FunctionType;\n arguments: Uint8Array;\n output_schema: Uint8Array;\n parameter_default_values?: /** @record-batch */ Uint8Array | null;\n stability?: FunctionStability | null;\n null_handling?: NullHandling | null;\n argument_monotonicity?: ArgumentMonotonicity[] | null;\n description: string;\n examples: CatalogExample[];\n categories: string[];\n projection_pushdown?: boolean | null;\n filter_pushdown?: boolean | null;\n sampling_pushdown?: boolean | null;\n late_materialization?: boolean | null;\n filter_semantic_profiles: string[];\n additional_filter_functions: FilterFunctionCapability[];\n runtime_filter_algorithms: RuntimeFilterAlgorithmCapability[];\n filter_evaluation_contexts: EvaluationContextCapability[];\n order_preservation?: OrderPreservation | null;\n max_workers?: number | null;\n supports_batch_index: boolean;\n supports_splits: boolean;\n filters_exactly_applied: boolean;\n supports_positions: boolean;\n split_token_ttl_seconds?: number | null;\n partition_kind: PartitionKind;\n order_dependent: OrderDependence;\n distinct_dependent: DistinctDependence;\n supports_window: boolean;\n streaming_partitioned: boolean;\n has_finalize: boolean;\n source_order_dependent: boolean;\n sink_order_dependent: boolean;\n requires_input_batch_index: boolean;\n input_from_args: boolean;\n required_settings: string[];\n required_secrets: SecretLookupEntry[];\n}\n\nexport interface MacroInfo {\n comment?: string | null;\n tags: Record<string, string>;\n name: string;\n schema_path: string[];\n macro_type: MacroType;\n parameters: string[];\n parameter_default_values?: /** @record-batch */ Uint8Array | null;\n definition: string;\n arguments_schema?: /** @arrow-schema */ Uint8Array | null;\n}\n\nexport interface IndexInfo {\n comment?: string | null;\n tags: Record<string, string>;\n name: string;\n schema_path: string[];\n table_name: string;\n index_type: string;\n constraint_type: IndexConstraintType;\n expressions: string[];\n options: Record<string, string>;\n}\n\nexport interface AggregateBindRequest {\n function_name: string;\n arguments: /** @vgi-arguments */ Uint8Array;\n input_schema?: /** @arrow-schema */ Uint8Array | null;\n settings?: /** @record-batch */ Uint8Array | null;\n secrets?: /** @record-batch */ Uint8Array | null;\n attach_opaque_data?: Uint8Array | null;\n schema_path?: string[] | null;\n argument_names?: (string | null)[] | null;\n}\n\nexport interface AggregateBindResponse {\n output_schema: /** @arrow-schema */ Uint8Array;\n execution_id: Uint8Array;\n}\n\nexport interface AggregateCombineRequest {\n function_name: string;\n execution_id: Uint8Array;\n merge_batch: Uint8Array;\n attach_opaque_data?: Uint8Array | null;\n schema_path?: string[] | null;\n}\n\nexport interface AggregateCombineResponse {\n}\n\nexport interface AggregateDestructorRequest {\n function_name: string;\n execution_id: Uint8Array;\n group_ids_batch: Uint8Array;\n attach_opaque_data?: Uint8Array | null;\n schema_path?: string[] | null;\n}\n\nexport interface AggregateDestructorResponse {\n}\n\nexport interface AggregateFinalizeRequest {\n function_name: string;\n execution_id: Uint8Array;\n group_ids_batch: Uint8Array;\n output_schema: /** @arrow-schema */ Uint8Array;\n attach_opaque_data?: Uint8Array | null;\n schema_path?: string[] | null;\n}\n\nexport interface AggregateFinalizeResponse {\n result_batch: Uint8Array;\n}\n\nexport interface AggregateStreamingChunkRequest {\n function_name: string;\n execution_id: Uint8Array;\n input_batch: Uint8Array;\n attach_opaque_data?: Uint8Array | null;\n schema_path?: string[] | null;\n}\n\nexport interface AggregateStreamingChunkResponse {\n result_batch: Uint8Array;\n}\n\nexport interface AggregateStreamingCloseRequest {\n function_name: string;\n execution_id: Uint8Array;\n attach_opaque_data?: Uint8Array | null;\n schema_path?: string[] | null;\n}\n\nexport interface AggregateStreamingCloseResponse {\n}\n\nexport interface AggregateStreamingOpenRequest {\n function_name: string;\n arguments: /** @vgi-arguments */ Uint8Array;\n input_schema: /** @arrow-schema */ Uint8Array;\n partition_key_count: number;\n order_key_count: number;\n output_schema: /** @arrow-schema */ Uint8Array;\n settings?: /** @record-batch */ Uint8Array | null;\n secrets?: /** @record-batch */ Uint8Array | null;\n attach_opaque_data?: Uint8Array | null;\n schema_path?: string[] | null;\n}\n\nexport interface AggregateStreamingOpenResponse {\n execution_id: Uint8Array;\n}\n\nexport interface AggregateUpdateRequest {\n function_name: string;\n execution_id: Uint8Array;\n input_batch: Uint8Array;\n attach_opaque_data?: Uint8Array | null;\n schema_path?: string[] | null;\n}\n\nexport interface AggregateUpdateResponse {\n}\n\nexport interface AggregateWindowRequest {\n function_name: string;\n execution_id: Uint8Array;\n partition_id: number;\n rid: number;\n frame_starts: number[];\n frame_ends: number[];\n attach_opaque_data?: Uint8Array | null;\n schema_path?: string[] | null;\n}\n\nexport interface AggregateWindowResponse {\n result_batch: Uint8Array;\n}\n\nexport interface AggregateWindowBatchRequest {\n function_name: string;\n execution_id: Uint8Array;\n partition_id: number;\n row_idx: number;\n count: number;\n frames_per_row: number[];\n frame_starts: number[];\n frame_ends: number[];\n attach_opaque_data?: Uint8Array | null;\n schema_path?: string[] | null;\n}\n\nexport interface AggregateWindowBatchResponse {\n result_batch: Uint8Array;\n}\n\nexport interface AggregateWindowDestructorRequest {\n function_name: string;\n execution_id: Uint8Array;\n partition_id: number;\n attach_opaque_data?: Uint8Array | null;\n schema_path?: string[] | null;\n}\n\nexport interface AggregateWindowDestructorResponse {\n}\n\nexport interface AggregateWindowInitRequest {\n function_name: string;\n execution_id: Uint8Array;\n partition_id: number;\n row_count: number;\n partition_batch: Uint8Array;\n output_schema: /** @arrow-schema */ Uint8Array;\n filter_mask: Uint8Array;\n frame_stats: Uint8Array;\n all_valid: Uint8Array;\n attach_opaque_data?: Uint8Array | null;\n schema_path?: string[] | null;\n}\n\nexport interface AggregateWindowInitResponse {\n}\n\nexport interface CopyFromContext {\n format: string;\n file_path: string;\n expected_schema: /** @arrow-schema */ Uint8Array;\n}\n\nexport interface CopyToContext {\n format: string;\n file_path: string;\n}\n\nexport interface BindRequest {\n function_name: string;\n arguments: /** @vgi-arguments */ Uint8Array;\n function_type: InvocationFunctionType;\n input_schema?: /** @arrow-schema */ Uint8Array | null;\n settings?: /** @record-batch */ Uint8Array | null;\n secrets?: /** @record-batch */ Uint8Array | null;\n attach_opaque_data?: Uint8Array | null;\n transaction_opaque_data?: Uint8Array | null;\n resolved_secrets_provided: boolean;\n at_unit?: string | null;\n at_value?: string | null;\n copy_from?: CopyFromContext | null;\n copy_to?: CopyToContext | null;\n schema_path?: string[] | null;\n argument_names?: (string | null)[] | null;\n}\n\nexport interface BindResponse {\n output_schema: /** @arrow-schema */ Uint8Array;\n opaque_data?: Uint8Array | null;\n lookup_secret_types: string[];\n lookup_scopes: string[];\n lookup_names: string[];\n}\n\nexport interface ClientCapabilities {\n engine: string;\n native_formats: string[];\n catalogs: string[];\n can_stream: boolean;\n filter_encodings: string[];\n}\n\nexport interface CatalogAttachRequest {\n name: string;\n options?: /** @record-batch */ Uint8Array | null;\n data_version_spec?: string | null;\n implementation_version?: string | null;\n client_capabilities?: ClientCapabilities | null;\n}\n\nexport interface CatalogAttachResult {\n attach_opaque_data: Uint8Array;\n supports_transactions: boolean;\n supports_time_travel: boolean;\n catalog_version_frozen: boolean;\n catalog_version: number;\n attach_opaque_data_required: boolean;\n default_schema: string;\n settings: Uint8Array[];\n secret_types: Uint8Array[];\n attach_catalogs: Uint8Array[];\n comment?: string | null;\n tags: Record<string, string>;\n supports_column_statistics: boolean;\n global_functions: Uint8Array[];\n global_function_prefix: string;\n resolved_data_version?: string | null;\n resolved_implementation_version?: string | null;\n supports_catalog_contents: boolean;\n}\n\nexport interface CatalogsResponse {\n items: CatalogInfo[];\n}\n\nexport interface SchemaContents {\n path: string[];\n schema: Uint8Array;\n tables: Uint8Array[];\n views: Uint8Array[];\n scalar_functions: Uint8Array[];\n aggregate_functions: Uint8Array[];\n table_functions: Uint8Array[];\n scalar_macros: Uint8Array[];\n table_macros: Uint8Array[];\n indexes: Uint8Array[];\n}\n\nexport interface CatalogContentsResponse {\n catalog_version: number;\n etag?: string | null;\n not_modified: boolean;\n schemas: SchemaContents[];\n}\n\nexport interface CopyFromFormatsResponse {\n items: Uint8Array[];\n}\n\nexport interface CatalogCreateRequest {\n name: string;\n on_conflict: OnConflict;\n options?: /** @record-batch */ Uint8Array | null;\n}\n\nexport interface IndexCreateRequest {\n attach_opaque_data: Uint8Array;\n schema_path: string[];\n name: string;\n table_name: string;\n index_type: string;\n constraint_type: IndexConstraintType;\n expressions: string[];\n on_conflict: OnConflict;\n options: Record<string, string>;\n transaction_opaque_data?: Uint8Array | null;\n}\n\nexport interface IndexesResponse {\n items: IndexInfo[];\n}\n\nexport interface MacroCreateRequest {\n attach_opaque_data: Uint8Array;\n schema_path: string[];\n name: string;\n macro_type: MacroType;\n parameters: string[];\n definition: string;\n on_conflict: OnConflict;\n parameter_default_values?: /** @record-batch */ Uint8Array | null;\n arguments_schema?: /** @arrow-schema */ Uint8Array | null;\n transaction_opaque_data?: Uint8Array | null;\n}\n\nexport interface MacrosResponse {\n items: MacroInfo[];\n}\n\nexport interface FunctionsResponse {\n items: FunctionInfo[];\n}\n\nexport interface TablesResponse {\n items: TableInfo[];\n}\n\nexport interface ViewsResponse {\n items: ViewInfo[];\n}\n\nexport interface SchemasResponse {\n items: SchemaInfo[];\n}\n\nexport interface TableCreateRequest {\n attach_opaque_data: Uint8Array;\n schema_path: string[];\n name: string;\n columns: Uint8Array;\n on_conflict: OnConflict;\n not_null_constraints: number[];\n unique_constraints: number[][];\n check_constraints: string[];\n primary_key_constraints: number[][];\n foreign_key_constraints: Uint8Array[];\n transaction_opaque_data?: Uint8Array | null;\n}\n\nexport interface TransactionBeginResponse {\n transaction_opaque_data?: Uint8Array | null;\n}\n\nexport interface CatalogVersionResponse {\n version: number;\n}\n\nexport interface InitRequest {\n bind_call: BindRequest;\n output_schema: /** @arrow-schema */ Uint8Array;\n bind_opaque_data?: Uint8Array | null;\n projection_ids?: number[] | null;\n pushdown_filters?: /** @record-batch */ Uint8Array | null;\n join_keys?: /** @record-batch */ Uint8Array[] | null;\n split_tokens?: Uint8Array[] | null;\n row_limit?: number | null;\n phase?: TableInOutFunctionInitPhase | null;\n finalize_state_id?: Uint8Array | null;\n execution_id?: Uint8Array | null;\n init_opaque_data?: Uint8Array | null;\n substream_id?: Uint8Array | null;\n order_by_column_name?: string | null;\n order_by_direction?: OrderByDirection | null;\n order_by_null_order?: OrderByNullOrder | null;\n order_by_limit?: number | null;\n tablesample_percentage?: number | null;\n tablesample_seed?: number | null;\n}\n\nexport interface TableBufferingCombineRequest {\n function_name: string;\n execution_id: Uint8Array;\n state_ids: Uint8Array[];\n attach_opaque_data?: Uint8Array | null;\n transaction_id?: Uint8Array | null;\n schema_path?: string[] | null;\n}\n\nexport interface TableBufferingCombineResponse {\n finalize_state_ids: Uint8Array[];\n}\n\nexport interface TableBufferingDestructorRequest {\n function_name: string;\n execution_id: Uint8Array;\n attach_opaque_data?: Uint8Array | null;\n transaction_id?: Uint8Array | null;\n schema_path?: string[] | null;\n}\n\nexport interface TableBufferingDestructorResponse {\n}\n\nexport interface TableBufferingProcessRequest {\n function_name: string;\n execution_id: Uint8Array;\n input_batch: Uint8Array;\n attach_opaque_data?: Uint8Array | null;\n transaction_id?: Uint8Array | null;\n batch_index?: number | null;\n schema_path?: string[] | null;\n}\n\nexport interface TableBufferingProcessResponse {\n state_id: Uint8Array;\n}\n\nexport interface TableFunctionCardinalityRequest {\n bind_call: BindRequest;\n bind_opaque_data?: Uint8Array | null;\n}\n\nexport interface TableCardinality {\n estimate?: number | null;\n max?: number | null;\n}\n\nexport interface TableFunctionDynamicToStringRequest {\n bind_call: BindRequest;\n bind_opaque_data?: Uint8Array | null;\n global_execution_id: Uint8Array;\n}\n\nexport interface TableFunctionDynamicToStringResponse {\n keys: string[];\n values: string[];\n}\n\nexport interface TableFunctionPlanRequest {\n bind_call: BindRequest;\n bind_opaque_data?: Uint8Array | null;\n projection_ids?: number[] | null;\n pushdown_filters?: /** @record-batch */ Uint8Array | null;\n join_keys?: /** @record-batch */ Uint8Array[] | null;\n row_limit?: number | null;\n target_split_bytes?: number | null;\n min_splits?: number | null;\n max_splits_per_response?: number | null;\n cursor?: Uint8Array | null;\n refined_filters?: /** @record-batch */ Uint8Array | null;\n filters_complete: boolean;\n start_position?: Uint8Array | null;\n end_position?: Uint8Array | null;\n order_by_column_name?: string | null;\n order_by_direction?: OrderByDirection | null;\n order_by_null_order?: OrderByNullOrder | null;\n order_by_limit?: number | null;\n tablesample_percentage?: number | null;\n tablesample_seed?: number | null;\n}\n\nexport interface ScanSplit {\n payload: Uint8Array;\n token: Uint8Array;\n estimated_rows?: number | null;\n rows_exact: boolean;\n estimated_bytes?: number | null;\n partition_bounds?: /** @record-batch */ Uint8Array | null;\n column_statistics?: Uint8Array | null;\n location_ids?: number[] | null;\n start_position?: Uint8Array | null;\n end_position?: Uint8Array | null;\n}\n\nexport interface PlanResponse {\n splits: ScanSplit[] | Uint8Array[];\n next_cursors?: Uint8Array[] | null;\n execution_id?: Uint8Array | null;\n init_opaque_data?: Uint8Array | null;\n max_workers?: number | null;\n estimated_total_splits?: number | null;\n estimated_total_rows?: number | null;\n estimated_total_bytes?: number | null;\n catalog_version?: number | null;\n scope: string;\n locations?: string[] | null;\n partitioning?: Uint8Array[] | null;\n sort_order?: Uint8Array[] | null;\n cache_max_age_seconds?: number | null;\n start_position?: Uint8Array | null;\n end_position?: Uint8Array | null;\n}\n\nexport interface TableFunctionStatisticsRequest {\n bind_call: BindRequest;\n bind_opaque_data?: Uint8Array | null;\n}\n\n// ----------------------------------------------------------------------------\n// Codec wrappers: turn the typed interfaces above into actual wire bytes.\n// ----------------------------------------------------------------------------\n\nexport const encodeCatalogInfo = (v: CatalogInfo): Uint8Array => encodeASD(CatalogInfoSchema, v);\nexport const decodeCatalogInfo = (b: Uint8Array): CatalogInfo => decodeASD<CatalogInfo>(CatalogInfoSchema, b);\nexport const encodeFunctionInfo = (v: FunctionInfo): Uint8Array => encodeASD(FunctionInfoSchema, v);\nexport const decodeFunctionInfo = (b: Uint8Array): FunctionInfo => decodeASD<FunctionInfo>(FunctionInfoSchema, b);\nexport const encodeIndexInfo = (v: IndexInfo): Uint8Array => encodeASD(IndexInfoSchema, v);\nexport const decodeIndexInfo = (b: Uint8Array): IndexInfo => decodeASD<IndexInfo>(IndexInfoSchema, b);\nexport const encodeMacroInfo = (v: MacroInfo): Uint8Array => encodeASD(MacroInfoSchema, v);\nexport const decodeMacroInfo = (b: Uint8Array): MacroInfo => decodeASD<MacroInfo>(MacroInfoSchema, b);\nexport const encodeSchemaInfo = (v: SchemaInfo): Uint8Array => encodeASD(SchemaInfoSchema, v);\nexport const decodeSchemaInfo = (b: Uint8Array): SchemaInfo => decodeASD<SchemaInfo>(SchemaInfoSchema, b);\nexport const encodeTableInfo = (v: TableInfo): Uint8Array => encodeASD(TableInfoSchema, v);\nexport const decodeTableInfo = (b: Uint8Array): TableInfo => decodeASD<TableInfo>(TableInfoSchema, b);\nexport const encodeViewInfo = (v: ViewInfo): Uint8Array => encodeASD(ViewInfoSchema, v);\nexport const decodeViewInfo = (b: Uint8Array): ViewInfo => decodeASD<ViewInfo>(ViewInfoSchema, b);\n",
|
|
57
57
|
"// ============================================================================\n// GENERATED by vgi.codegen.ts_types. DO NOT EDIT BY HAND.\n//\n// Generator: python -m vgi.codegen.ts_types v1\n// Content hash: d240510f013b\n//\n// To regenerate:\n// uv run --project ~/Development/vgi-python python scripts/regen_generated.py\n// ============================================================================\n\nimport { encodeASD, decodeASD } from \"../codec/asd.js\";\nimport {\n CatalogAttachResultSchema,\n CatalogContentsResultSchema,\n SchemaContentsSchema,\n} from \"./vgi-protocol-schemas.js\";\nimport type { CatalogAttachResult, CatalogContentsResponse, SchemaContents } from \"./vgi-client.js\";\nexport type { CatalogAttachResult, CatalogContentsResponse, SchemaContents };\n\n// ----------------------------------------------------------------------------\n// CatalogAttachResult (schema: CatalogAttachResultSchema)\n// ----------------------------------------------------------------------------\n\n/** Fields of `CatalogAttachResult` that have a protocol default. */\nexport type CatalogAttachResultDefaultedField =\n | \"attach_opaque_data_required\"\n | \"default_schema\"\n | \"settings\"\n | \"secret_types\"\n | \"attach_catalogs\"\n | \"comment\"\n | \"tags\"\n | \"supports_column_statistics\"\n | \"global_functions\"\n | \"global_function_prefix\"\n | \"supports_catalog_contents\";\n/** `CatalogAttachResult` with every field that has a protocol default optional. */\nexport type CatalogAttachResultInit = Omit<CatalogAttachResult, CatalogAttachResultDefaultedField> & Partial<Pick<CatalogAttachResult, CatalogAttachResultDefaultedField>>;\n\n/** Build a complete `CatalogAttachResult`, filling the protocol defaults, in wire order. */\nexport function buildCatalogAttachResult(init: CatalogAttachResultInit): CatalogAttachResult {\n return {\n attach_opaque_data: init.attach_opaque_data,\n supports_transactions: init.supports_transactions,\n supports_time_travel: init.supports_time_travel,\n catalog_version_frozen: init.catalog_version_frozen,\n catalog_version: init.catalog_version,\n attach_opaque_data_required: init.attach_opaque_data_required ?? true,\n default_schema: init.default_schema ?? \"main\",\n settings: init.settings ?? [],\n secret_types: init.secret_types ?? [],\n attach_catalogs: init.attach_catalogs ?? [],\n comment: init.comment ?? null,\n tags: init.tags ?? {},\n supports_column_statistics: init.supports_column_statistics ?? false,\n global_functions: init.global_functions ?? [],\n global_function_prefix: init.global_function_prefix ?? \"\",\n resolved_data_version: init.resolved_data_version ?? null,\n resolved_implementation_version: init.resolved_implementation_version ?? null,\n supports_catalog_contents: init.supports_catalog_contents ?? false,\n };\n}\n\nexport const encodeCatalogAttachResult = (v: CatalogAttachResult): Uint8Array => encodeASD(CatalogAttachResultSchema, v);\nexport const decodeCatalogAttachResult = (b: Uint8Array): CatalogAttachResult => decodeASD<CatalogAttachResult>(CatalogAttachResultSchema, b);\n\n// ----------------------------------------------------------------------------\n// SchemaContents (schema: SchemaContentsSchema)\n// ----------------------------------------------------------------------------\n\n/** Fields of `SchemaContents` that have a protocol default. */\nexport type SchemaContentsDefaultedField =\n | \"tables\"\n | \"views\"\n | \"scalar_functions\"\n | \"aggregate_functions\"\n | \"table_functions\"\n | \"scalar_macros\"\n | \"table_macros\"\n | \"indexes\";\n/** `SchemaContents` with every field that has a protocol default optional. */\nexport type SchemaContentsInit = Omit<SchemaContents, SchemaContentsDefaultedField> & Partial<Pick<SchemaContents, SchemaContentsDefaultedField>>;\n\n/** Build a complete `SchemaContents`, filling the protocol defaults, in wire order. */\nexport function buildSchemaContents(init: SchemaContentsInit): SchemaContents {\n return {\n path: init.path,\n schema: init.schema,\n tables: init.tables ?? [],\n views: init.views ?? [],\n scalar_functions: init.scalar_functions ?? [],\n aggregate_functions: init.aggregate_functions ?? [],\n table_functions: init.table_functions ?? [],\n scalar_macros: init.scalar_macros ?? [],\n table_macros: init.table_macros ?? [],\n indexes: init.indexes ?? [],\n };\n}\n\nexport const encodeSchemaContents = (v: SchemaContents): Uint8Array => encodeASD(SchemaContentsSchema, v);\nexport const decodeSchemaContents = (b: Uint8Array): SchemaContents => decodeASD<SchemaContents>(SchemaContentsSchema, b);\n\n// ----------------------------------------------------------------------------\n// CatalogContentsResponse (schema: CatalogContentsResultSchema)\n// ----------------------------------------------------------------------------\n\n/** Fields of `CatalogContentsResponse` that have a protocol default. */\nexport type CatalogContentsResponseDefaultedField =\n | \"etag\"\n | \"not_modified\"\n | \"schemas\";\n/** `CatalogContentsResponse` with every field that has a protocol default optional. */\nexport type CatalogContentsResponseInit = Omit<CatalogContentsResponse, CatalogContentsResponseDefaultedField> & Partial<Pick<CatalogContentsResponse, CatalogContentsResponseDefaultedField>>;\n\n/** Build a complete `CatalogContentsResponse`, filling the protocol defaults, in wire order. */\nexport function buildCatalogContentsResponse(init: CatalogContentsResponseInit): CatalogContentsResponse {\n return {\n catalog_version: init.catalog_version,\n etag: init.etag ?? null,\n not_modified: init.not_modified ?? false,\n schemas: init.schemas ?? [],\n };\n}\n\nexport const encodeCatalogContentsResponse = (v: CatalogContentsResponse): Uint8Array => encodeASD(CatalogContentsResultSchema, v);\nexport const decodeCatalogContentsResponse = (b: Uint8Array): CatalogContentsResponse => decodeASD<CatalogContentsResponse>(CatalogContentsResultSchema, b);\n",
|
|
58
58
|
"// Copyright 2025, 2026 Query Farm LLC - https://query.farm\n// Catalog items that are built once and encoded once.\n//\n// A function listing from a descriptor-driven catalog depends only on static\n// metadata, yet every catalog_schema_contents_functions request used to rebuild\n// each FunctionInfo (argument and output schemas serialized to IPC, metadata\n// resolved) and then Arrow-encode it again: ~1.5 ms per function, ~236 ms for\n// the example worker's main schema, paid on every function-set load DuckDB\n// makes. vgi-python fixed the same cost by caching the listing and storing each\n// item's encoding on its (frozen dataclass) instance (6cc522a).\n//\n// JS objects are not frozen by default, and an encoding memo keyed by object\n// identity is only sound for an object that cannot change after it was\n// encoded. So the memo applies to FROZEN items only: a catalog opts in by\n// returning items it froze with `freezeCatalogItem` (ReadOnlyCatalogInterface\n// does), and every other item -- a catalog that builds fresh objects per call,\n// or keeps and mutates its own -- is encoded on every call exactly as before.\n\nimport { encodeFunctionInfo, type FunctionInfo } from \"../generated/vgi-client.js\";\n\n/** Encoded bytes per frozen item instance. Weak, so it never pins an item. */\nconst ENCODED = new WeakMap<object, Uint8Array>();\n\n/**\n * Deep-freeze a catalog item (plain objects and arrays, recursively) so it can\n * be shared between requests and its encoding memoized.\n *\n * Binary fields (`Uint8Array` and other ArrayBuffer views) are left as they are\n * -- a typed array with elements cannot be frozen -- and must be treated as\n * read-only by convention, like the rest of a shared item.\n */\nexport function freezeCatalogItem<T extends object>(item: T): Readonly<T> {\n deepFreeze(item);\n return item;\n}\n\nfunction deepFreeze(value: unknown): void {\n if (value === null || typeof value !== \"object\" || Object.isFrozen(value) || ArrayBuffer.isView(value)) return;\n const proto = Object.getPrototypeOf(value);\n if (proto !== Object.prototype && proto !== Array.prototype && proto !== null) return;\n for (const child of Object.values(value)) deepFreeze(child);\n Object.freeze(value);\n}\n\n/**\n * Arrow-encode a FunctionInfo, once per frozen instance.\n *\n * The bytes returned for a frozen item are shared between calls; callers copy\n * them into a response and never write to them.\n */\nexport function encodeFunctionInfoOnce(info: FunctionInfo): Uint8Array {\n if (!Object.isFrozen(info)) return encodeFunctionInfo(info);\n let encoded = ENCODED.get(info);\n if (encoded === undefined) {\n encoded = encodeFunctionInfo(info);\n ENCODED.set(info, encoded);\n }\n return encoded;\n}\n",
|