@volter/world-core 2.0.36 → 3.0.0

package/dist/src/anthropic-wire.js

@@ -0,0 +1,136 @@
+// Anthropic's Messages wire (architecture, "The kernel owns shared mechanics"): the Messages shapes a vendor serves as
+// Anthropic defines them (https://platform.claude.com/docs/en/api/messages and /build-with-claude/streaming), for
+// generative packs that run no model, beside the OpenAI wire (./openai-wire.ts) whose turn vocabulary it answers. A
+// pack decides its turn (the World's scenario's, else the stub) and its own facts (its ids, its usage fields); this
+// module reads a request as the scenario reads it and shapes the answer and its Server-Sent Events. Importing it does
+// nothing.
+import { estimateTokens, forbidsTools } from "./openai-wire.js";
+import { sampleFromSchema } from "./schema-sample.js";
+const obj = (v) => (v && typeof v === 'object' && !Array.isArray(v) ? v : {});
+const arr = (v) => (Array.isArray(v) ? v : []);
+/** A message's content as text: a string, or its text blocks (and the text of its tool results) joined. */
+export const messagesText = (content) => typeof content === 'string' ? content : arr(content).map((c) => (c.type === 'text' ? String(c.text ?? '') : c.type === 'tool_result' ? messagesText(c.content) : '')).filter(Boolean).join('\n');
+/** The last user message's text. */
+export function messagesLastUserText(messages) {
+    for (let i = messages.length - 1; i >= 0; i--)
+        if (messages[i].role === 'user') {
+            const t = messagesText(messages[i].content);
+            if (t)
+                return t;
+        }
+    return '';
+}
+/** The tool results the last message carries, by the names of the tool_use blocks they answer. */
+export function messagesToolResults(messages) {
+    const ids = arr(messages.at(-1)?.content).filter((c) => c.type === 'tool_result').map((c) => c.tool_use_id);
+    if (!ids.length)
+        return [];
+    return messages.flatMap((m) => arr(m.content)).filter((c) => c.type === 'tool_use' && ids.includes(c.id)).map((c) => String(c.name));
+}
+/** A Messages request as a scenario reads it (the OpenAI wire's `WireRequest`): its model, its system and messages'
+ *  text, its last user text, the tools it offers and the tool results it ends with. */
+export function messagesWireRequest(body) {
+    const messages = arr(body.messages);
+    const system = typeof body.system === 'string' ? body.system : messagesText(body.system);
+    return {
+        model: String(body.model ?? ''), text: [system, ...messages.map((m) => messagesText(m.content))].join('\n'), lastUser: messagesLastUserText(messages),
+        tools: arr(body.tools).map((t) => String(t.name ?? '')).filter(Boolean), results: messagesToolResults(messages),
+        ...(forbidsTools(body.tool_choice) ? { toolsForbidden: true } : {}),
+    };
+}
+/** The labeled stub for a Messages request: with tools offered (`tool_choice` not `none`) and the last message not a
+ *  tool's result, it calls the tool the choice names, else the first, with input its schema admits; with a JSON schema
+ *  for the output, it answers a value the schema admits; otherwise it echoes the last user message. `label` names the
+ *  twin. */
+export function messagesStubTurn(body, label) {
+    const model = String(body.model);
+    const messages = arr(body.messages);
+    const tools = arr(body.tools).filter((t) => t.input_schema !== undefined);
+    const choice = obj(body.tool_choice ?? { type: 'auto' });
+    if (tools.length && choice.type !== 'none' && (messagesToolResults(messages).length === 0 || choice.type === 'any' || choice.type === 'tool')) {
+        const tool = (choice.type === 'tool' ? tools.find((t) => t.name === choice.name) : undefined) ?? tools[0];
+        return { toolCalls: [{ name: String(tool.name), arguments: sampleFromSchema(tool.input_schema, String(tool.name)) }] };
+    }
+    const format = obj(obj(body.output_config).format ?? body.output_format);
+    if (format.type === 'json_schema')
+        return { text: JSON.stringify(sampleFromSchema(format.schema)) };
+    const said = messagesLastUserText(messages).trim();
+    return { text: `[twin-stub:${model}] This is a deterministic stub from the ${label} (no model runs). Your last message: ${said.length > 200 ? `${said.slice(0, 200)}…` : said || '(empty)'}` };
+}
+/** The OpenAI wire's finish reasons as the Messages stop reasons they mean. */
+const STOP = { stop: 'end_turn', length: 'max_tokens', tool_calls: 'tool_use', content_filter: 'refusal' };
+/** A message (`type: message`): a thinking block when the request turns thinking on, then the
+ *  text, then the tool_use blocks; its stop reason, and its usage estimated from the request and the answer. A text
+ *  longer than `max_tokens` is cut there and stops `max_tokens`. */
+export function message(s) {
+    const b = s.body;
+    // thinking blocks come only when the request turns thinking on; a turn's reasoning is then their text
+    const thinkingOn = ['enabled', 'adaptive'].includes(String(obj(b.thinking).type));
+    const max = typeof b.max_tokens === 'number' ? b.max_tokens : undefined;
+    let text = s.turn.text ?? undefined;
+    let cut = false;
+    if (text !== undefined && max !== undefined && estimateTokens(text) > max) {
+        text = text.slice(0, Math.max(1, max) * 4);
+        cut = true;
+    }
+    const content = [];
+    if (thinkingOn) {
+        const thinking = s.turn.reasoning ?? '[twin-stub] No model runs here, so there is no reasoning to show.';
+        content.push({ type: 'thinking', thinking, signature: s.signature ? s.signature(thinking) : '' });
+    }
+    if (text !== undefined)
+        content.push({ type: 'text', text });
+    (s.turn.toolCalls ?? []).forEach((c, i) => content.push({ type: 'tool_use', id: s.toolId(i), name: c.name, input: c.arguments ?? {} }));
+    const stop = cut ? 'max_tokens' : s.turn.finishReason ? (STOP[s.turn.finishReason] ?? s.turn.finishReason) : s.turn.toolCalls?.length ? 'tool_use' : 'end_turn';
+    const request = messagesWireRequest(b);
+    const output = estimateTokens(content.map((c) => (c.type === 'tool_use' ? JSON.stringify(c.input) : String(c.text ?? c.thinking ?? ''))).join(''));
+    return {
+        id: s.id, type: 'message', role: 'assistant', model: s.model, content, stop_reason: stop, stop_sequence: null,
+        usage: { input_tokens: estimateTokens(request.text), cache_creation_input_tokens: 0, cache_read_input_tokens: 0, output_tokens: output, ...(s.usage ?? {}) },
+    };
+}
+/** Text in the pieces a stream sends it in: a few words at a time. */
+function pieces(text) {
+    const words = text.match(/\S+\s*|\s+/g) ?? [text];
+    const out = [];
+    for (let i = 0; i < words.length; i += 4)
+        out.push(words.slice(i, i + 4).join(''));
+    return out.length ? out : [''];
+}
+/** A message as the stream's events: message_start (its content empty), per block its start, deltas and stop (a ping
+ *  after the first start), message_delta with the stop reason and the output usage, message_stop. `extra` adds fields
+ *  a vendor puts on the two closing events (Merge's `routing`). */
+export function messageSSE(m, extra = {}, options = {}) {
+    const events = [];
+    const send = (type, data) => { events.push(`event: ${type}\ndata: ${JSON.stringify({ type, ...data })}\n\n`); };
+    const usage = obj(m.usage);
+    send('message_start', { message: { ...m, content: [], stop_reason: null, stop_sequence: null, usage: { ...usage, output_tokens: 1 } } });
+    arr(m.content).forEach((block, index) => {
+        if (block.type === 'text') {
+            send('content_block_start', { index, content_block: { type: 'text', text: '' } });
+            if (index === 0 && options.ping !== false)
+                send('ping', {});
+            for (const piece of pieces(String(block.text)))
+                send('content_block_delta', { index, delta: { type: 'text_delta', text: piece } });
+        }
+        else if (block.type === 'thinking') {
+            send('content_block_start', { index, content_block: { type: 'thinking', thinking: '', signature: '' } });
+            if (index === 0 && options.ping !== false)
+                send('ping', {});
+            for (const piece of pieces(String(block.thinking)))
+                send('content_block_delta', { index, delta: { type: 'thinking_delta', thinking: piece } });
+            send('content_block_delta', { index, delta: { type: 'signature_delta', signature: block.signature } });
+        }
+        else {
+            send('content_block_start', { index, content_block: { ...block, input: {} } });
+            if (index === 0 && options.ping !== false)
+                send('ping', {});
+            for (const piece of options.completeToolInput ? [JSON.stringify(block.input)] : pieces(JSON.stringify(block.input)))
+                send('content_block_delta', { index, delta: { type: 'input_json_delta', partial_json: piece } });
+        }
+        send('content_block_stop', { index });
+    });
+    send('message_delta', { delta: { stop_reason: m.stop_reason, stop_sequence: null }, usage: { output_tokens: usage.output_tokens }, ...extra });
+    send('message_stop', { ...extra });
+    return events.join('');
+}

package/dist/src/bytes.d.ts

@@ -0,0 +1,7 @@
+/** The bytes of a gzip member (a SARIF upload, a tarball's contents, a gzipped request body). */
+export declare function gunzip(bytes: Uint8Array): Promise<Uint8Array>;
+/** Bytes as a gzip member. */
+export declare function gzip(bytes: Uint8Array): Promise<Uint8Array>;
+/** Bytes as a zlib (deflate) member: git's object and pack framing. Reading one back is git/inflate.ts, which also
+ *  reports how many bytes it consumed, since a packfile's objects sit one after another with no length. */
+export declare function deflate(bytes: Uint8Array): Promise<Uint8Array>;

package/dist/src/bytes.js

@@ -0,0 +1,35 @@
+// Bytes a vendor's wire carries compressed, as a handler reads them. Web-standard compression streams where the runtime
+// has them (workerd, the browser, Node), so the kernel's root keeps no static import of Node's zlib — a browser client
+// imports this module too. Bun has neither `CompressionStream` nor `DecompressionStream` (Bun 1.2.20:
+// `bun -e "typeof CompressionStream"` prints `undefined`), and a local World is served by Bun, so there the builtin is
+// reached the way the kernel reaches every builtin: `process.getBuiltinModule`, at call time, never at import (the seam
+// serve-http.ts's `nodeBuiltin` uses).
+function zlib(what) {
+    const get = globalThis.process?.getBuiltinModule;
+    if (typeof get !== 'function')
+        throw new Error(`${what}: this runtime has neither the compression streams nor node:zlib (process.getBuiltinModule)`);
+    return get('node:zlib');
+}
+async function streamed(bytes, stream) {
+    const piped = new Response(bytes).body.pipeThrough(stream);
+    return new Uint8Array(await new Response(piped).arrayBuffer());
+}
+/** The bytes of a gzip member (a SARIF upload, a tarball's contents, a gzipped request body). */
+export async function gunzip(bytes) {
+    if (typeof DecompressionStream === 'function')
+        return streamed(bytes, new DecompressionStream('gzip'));
+    return new Uint8Array(zlib('gunzip').gunzipSync(bytes));
+}
+/** Bytes as a gzip member. */
+export async function gzip(bytes) {
+    if (typeof CompressionStream === 'function')
+        return streamed(bytes, new CompressionStream('gzip'));
+    return new Uint8Array(zlib('gzip').gzipSync(bytes));
+}
+/** Bytes as a zlib (deflate) member: git's object and pack framing. Reading one back is git/inflate.ts, which also
+ *  reports how many bytes it consumed, since a packfile's objects sit one after another with no length. */
+export async function deflate(bytes) {
+    if (typeof CompressionStream === 'function')
+        return streamed(bytes, new CompressionStream('deflate'));
+    return new Uint8Array(zlib('deflate').deflateSync(bytes));
+}

package/dist/src/changeset.d.ts

@@ -155,7 +155,7 @@ export type Changeset = {
         contentHash: string;
         at: string;
     };
-    /** PROTOCOL 2 — where the parent log stood when this was cut, per state service (its entry count):
+    /** Where the parent log stood when this was cut, per state service (its entry count):
      *  a rebase reads the parent's entries since here to name what moved under the changeset. Outside
      *  the hash (metadata of the cut, not of the change). */
     cut?: Record<string, HistoryReference>;

package/dist/src/clickhouse/index.d.ts

@@ -0,0 +1,3 @@
+export * from './values.js';
+export * from './types.js';
+export * from './sql.js';

package/dist/src/clickhouse/index.js

@@ -0,0 +1,6 @@
+// The kernel's ClickHouse® SQL engine (architecture, "The kernel owns shared mechanics"): parsed into an AST and evaluated
+// over rows a pack supplies as a catalog of relations. Tinybird's pipes and PostHog's HogQL are ClickHouse SQL; each pack
+// keeps its own front (Tinybird's templates, HogQL's tables and property access). Importing it does nothing.
+export * from "./values.js";
+export * from "./types.js";
+export * from "./sql.js";

package/dist/src/clickhouse/sql.d.ts

@@ -0,0 +1,233 @@
+import { type IntervalUnit } from './values.js';
+import { type ClickHouseColumn } from './types.js';
+import { ClickHouseError } from './values.js';
+export { ClickHouseError };
+export type Expr = {
+    k: 'lit';
+    v: unknown;
+    type: string;
+    text: string;
+} | {
+    k: 'id';
+    parts: string[];
+} | {
+    k: 'star';
+    qualifier?: string;
+} | {
+    k: 'fn';
+    name: string;
+    args: Expr[];
+    params?: Expr[];
+    distinct?: boolean;
+} | {
+    k: 'lambda';
+    params: string[];
+    body: Expr;
+} | {
+    k: 'bin';
+    op: string;
+    a: Expr;
+    b: Expr;
+} | {
+    k: 'not';
+    a: Expr;
+} | {
+    k: 'neg';
+    a: Expr;
+} | {
+    k: 'isnull';
+    a: Expr;
+    not: boolean;
+} | {
+    k: 'between';
+    a: Expr;
+    lo: Expr;
+    hi: Expr;
+    not: boolean;
+} | {
+    k: 'case';
+    operand?: Expr;
+    whens: Array<[Expr, Expr]>;
+    else?: Expr;
+} | {
+    k: 'alias';
+    name: string;
+    e: Expr;
+} | {
+    k: 'subquery';
+    q: Query;
+} | {
+    k: 'array';
+    items: Expr[];
+} | {
+    k: 'tuple';
+    items: Expr[];
+} | {
+    k: 'index';
+    a: Expr;
+    i: Expr;
+} | {
+    k: 'interval';
+    n: Expr;
+    unit: IntervalUnit;
+} | {
+    k: 'cast';
+    e: Expr;
+    type: string;
+}
+/** A ClickHouse query parameter `{name: Type}` (what `tb_secret` renders at publish). */
+ | {
+    k: 'param';
+    name: string;
+    type: string;
+};
+type JoinKind = 'inner' | 'left' | 'right' | 'full' | 'cross';
+type Strictness = 'all' | 'any' | 'semi' | 'anti';
+export type TableSource = {
+    kind: 'table';
+    name: string;
+} | {
+    kind: 'sub';
+    query: Query;
+};
+export type FromItem = {
+    source: TableSource;
+    final: boolean;
+    alias?: string;
+    join?: {
+        kind: JoinKind;
+        strictness: Strictness;
+        on?: Expr;
+        using?: string[];
+    };
+};
+export type OrderItem = {
+    e: Expr;
+    desc: boolean;
+    nullsFirst: boolean;
+};
+export type WithItem = {
+    name: string;
+    expr?: Expr;
+    query?: Query;
+};
+export type Select = {
+    with: WithItem[];
+    distinct: boolean;
+    items: Expr[];
+    from: FromItem[];
+    prewhere?: Expr;
+    where?: Expr;
+    groupBy: Expr[];
+    having?: Expr;
+    orderBy: OrderItem[];
+    limit?: Expr;
+    offset?: Expr;
+};
+export type Query = {
+    selects: Select[];
+    format?: string;
+};
+/** Parse one SQL statement (a SELECT, possibly with UNION ALL and a trailing FORMAT). */
+export declare function parseSql(sql: string): Query;
+/** A relation; `declared` marks a Data Source's own columns (their types are schema, not inference). */
+export type Relation = {
+    columns: ClickHouseColumn[];
+    rows: unknown[][];
+    declared?: boolean;
+};
+/** Where a FROM name resolves: pipe nodes, Data Sources, CTEs. */
+export interface Catalog {
+    /** A named relation, or undefined when the name is unknown. `final` requests FINAL semantics. */
+    table(name: string, final: boolean): Relation | undefined;
+}
+type Source = {
+    alias?: string;
+    name?: string;
+    columns: ClickHouseColumn[];
+    index: Map<string, number>;
+    declared?: boolean;
+};
+type JRow = unknown[][];
+type Scope = {
+    sources: Source[];
+    usingKeys: Set<string>;
+    aliases: Map<string, Expr>;
+    withExprs: Map<string, Expr>;
+    withCache: Map<string, unknown>;
+    exec: Executor;
+    subCache: Map<Expr, unknown>;
+    outer?: Scope;
+};
+type Env = {
+    scope: Scope;
+    row: JRow;
+    group?: JRow[];
+    aliasCache: Map<string, unknown>;
+    evaluating: Set<string>;
+    lambda?: Map<string, unknown>;
+    aj?: Map<Expr, unknown>;
+};
+export declare function evalExpr(e: Expr, env: Env): unknown;
+export declare function isKnownFunction(name: string): boolean;
+export declare class Executor {
+    readonly catalog: Catalog;
+    readonly nowMs: number;
+    readonly validating: boolean;
+    rowsRead: number;
+    bytesRead: number;
+    /** `validating`: publish-time analysis — subqueries in expressions are analyzed even when no row reaches them. */
+    constructor(catalog: Catalog, nowMs: number, validating?: boolean);
+    /** Resolve a FROM name: CTEs in scope first, then the catalog. */
+    table(name: string, final: boolean, scope?: Scope): Relation;
+    run(q: Query, outer?: Scope): Relation;
+    /** Run and also report rows_before_limit (for the response envelope). */
+    runTop(q: Query): {
+        rel: Relation;
+        rowsBeforeLimit?: number;
+    };
+    private select;
+    /** The analyzer pass: every expression of the SELECT resolves (see "static checking"). */
+    private check;
+    private checkExpr;
+    private orderValue;
+    private defaultRow;
+    private relationOf;
+    private from;
+    private combine;
+}
+export type QueryResult = {
+    meta: ClickHouseColumn[];
+    data: Array<Record<string, unknown>>;
+    rows: number;
+    rowsBeforeLimitAtLeast?: number;
+    rowsRead: number;
+    bytesRead: number;
+};
+/** Execute a parsed query against a catalog and shape Tinybird's result envelope. */
+export declare function executeSql(q: Query, catalog: Catalog, nowMs: number, opts?: {
+    validate?: boolean;
+}): QueryResult;
+/** Run a query to a Relation (for materialized views and node chaining). */
+export declare function runSql(q: Query, catalog: Catalog, nowMs: number, opts?: {
+    validate?: boolean;
+}): Relation;
+/** Every table name a query reads (FROM / JOIN / IN-subqueries), excluding its own CTEs. */
+export declare function usedTables(q: Query): string[];
+/** The left-most table of the first SELECT — a materialized view's trigger table. */
+export declare function primaryTable(q: Query): string | undefined;
+export declare function queryStatistics(result: QueryResult): {
+    elapsed: number;
+    rows_read: number;
+    bytes_read: number;
+};
+/**
+ * Render a result in one of the modeled text formats (`.json` is handled by the caller):
+ * ClickHouse's JSONEachRow and CSV / CSVWithNames, every row (and the header) ending in `\n`.
+ */
+export declare function renderFormat(format: 'ndjson' | 'csv' | 'csvwithnames', result: QueryResult): string;
+export type QueryPlan = Query;
+/** Parse a SELECT and apply the checks ClickHouse makes before reading any data. */
+export declare function parseQuery(sql: string): Query;
+/** Run a query over ONE table's stored rows (any FROM name resolves to it). */
+export declare function executeQuery(q: Query, columns: readonly ClickHouseColumn[], rows: ReadonlyArray<Record<string, unknown>>, nowMs?: number): QueryResult;