@justram/pie 0.1.0

package/dist/core/schema/normalize.js

@@ -0,0 +1,187 @@
+const PROVIDERS_WITH_SCHEMA_CONSTRAINTS = new Set([
+    "google-antigravity",
+    "google-gemini-cli",
+]);
+const PROVIDERS_REQUIRING_OBJECT_SCHEMA = new Set([
+    "google-antigravity",
+    "google-gemini-cli",
+    "openai-codex",
+]);
+const WRAPPED_VALUE_KEY = "value";
+export function normalizeToolSchema(model, schema) {
+    let normalizedSchema = schema;
+    if (PROVIDERS_WITH_SCHEMA_CONSTRAINTS.has(model.provider)) {
+        const cloned = deepClone(schema);
+        const withoutRefs = resolveRefs(cloned);
+        const normalized = normalizeLiterals(withoutRefs);
+        normalizedSchema = stripUnsupportedKeywords(normalized);
+    }
+    if (!PROVIDERS_REQUIRING_OBJECT_SCHEMA.has(model.provider)) {
+        return { schema: normalizedSchema, unwrapKey: null };
+    }
+    return wrapNonObjectSchema(normalizedSchema);
+}
+function deepClone(value) {
+    if (typeof structuredClone === "function") {
+        return structuredClone(value);
+    }
+    return JSON.parse(JSON.stringify(value));
+}
+function resolveRefs(schema) {
+    if (!schema || typeof schema !== "object") {
+        return schema;
+    }
+    const root = schema;
+    const defs = root.$defs ?? root.definitions;
+    const seen = new Map();
+    const resolving = new Set();
+    const resolveNode = (node) => {
+        if (Array.isArray(node)) {
+            return node.map(resolveNode);
+        }
+        if (!node || typeof node !== "object") {
+            return node;
+        }
+        const obj = node;
+        const ref = typeof obj.$ref === "string" ? obj.$ref : null;
+        if (ref) {
+            const target = resolveRef(ref, defs);
+            if (target) {
+                if (seen.has(ref)) {
+                    return seen.get(ref);
+                }
+                if (resolving.has(ref)) {
+                    return obj;
+                }
+                resolving.add(ref);
+                const resolved = resolveNode(target);
+                resolving.delete(ref);
+                seen.set(ref, resolved);
+                return resolved;
+            }
+        }
+        const result = {};
+        for (const [key, value] of Object.entries(obj)) {
+            if (key === "$defs" || key === "definitions") {
+                continue;
+            }
+            result[key] = resolveNode(value);
+        }
+        return result;
+    };
+    return resolveNode(root);
+}
+function resolveRef(ref, defs) {
+    if (!defs) {
+        return null;
+    }
+    const prefix = ref.startsWith("#/$defs/") ? "#/$defs/" : ref.startsWith("#/definitions/") ? "#/definitions/" : null;
+    if (!prefix) {
+        return null;
+    }
+    const key = ref.slice(prefix.length);
+    return defs[key];
+}
+function normalizeLiterals(schema) {
+    if (Array.isArray(schema)) {
+        return schema.map(normalizeLiterals);
+    }
+    if (!schema || typeof schema !== "object") {
+        return schema;
+    }
+    const node = schema;
+    const anyOf = node.anyOf;
+    if (Array.isArray(anyOf) && anyOf.length > 0 && anyOf.every(isConstSchema)) {
+        const values = anyOf.map((item) => item.const);
+        const types = collectEnumTypes(anyOf);
+        const next = {};
+        for (const [key, value] of Object.entries(node)) {
+            if (key === "anyOf") {
+                continue;
+            }
+            next[key] = normalizeLiterals(value);
+        }
+        next.enum = values;
+        if (types.length === 1) {
+            next.type = types[0];
+        }
+        else if (types.length === 0) {
+            delete next.type;
+        }
+        return next;
+    }
+    if ("const" in node) {
+        const value = node.const;
+        const next = {};
+        for (const [key, val] of Object.entries(node)) {
+            if (key === "const") {
+                continue;
+            }
+            next[key] = normalizeLiterals(val);
+        }
+        next.enum = [value];
+        return next;
+    }
+    const result = {};
+    for (const [key, value] of Object.entries(node)) {
+        result[key] = normalizeLiterals(value);
+    }
+    return result;
+}
+function stripUnsupportedKeywords(schema) {
+    if (Array.isArray(schema)) {
+        return schema.map(stripUnsupportedKeywords);
+    }
+    if (!schema || typeof schema !== "object") {
+        return schema;
+    }
+    const node = schema;
+    const result = {};
+    for (const [key, value] of Object.entries(node)) {
+        if (key === "examples") {
+            continue;
+        }
+        result[key] = stripUnsupportedKeywords(value);
+    }
+    return result;
+}
+function wrapNonObjectSchema(schema) {
+    if (isObjectSchema(schema)) {
+        return { schema, unwrapKey: null };
+    }
+    const wrapped = {
+        type: "object",
+        properties: {
+            [WRAPPED_VALUE_KEY]: schema,
+        },
+        required: [WRAPPED_VALUE_KEY],
+        additionalProperties: false,
+    };
+    return { schema: wrapped, unwrapKey: WRAPPED_VALUE_KEY };
+}
+function isObjectSchema(schema) {
+    if (!schema || typeof schema !== "object") {
+        return false;
+    }
+    const node = schema;
+    if (node.type === "object") {
+        return true;
+    }
+    return "properties" in node;
+}
+function isConstSchema(node) {
+    return !!node && typeof node === "object" && "const" in node;
+}
+function collectEnumTypes(items) {
+    const types = new Set();
+    for (const item of items) {
+        if (!item || typeof item !== "object") {
+            return [];
+        }
+        const type = item.type;
+        if (typeof type === "string") {
+            types.add(type);
+        }
+    }
+    return Array.from(types);
+}

package/dist/core/setup.d.ts

@@ -0,0 +1,13 @@
+import type { ExtractOptions } from "./types.js";
+export interface LoadExtractionSetupOptions<T> {
+    vars?: Record<string, unknown>;
+    overrides?: Partial<Omit<ExtractOptions<T>, "prompt" | "schema">>;
+}
+export interface ExtractionSetup<T> {
+    path: string;
+    prompt: string;
+    frontmatter: Record<string, unknown>;
+    options: ExtractOptions<T>;
+}
+export declare function loadExtractionSetup<T>(setupPath: string, options?: LoadExtractionSetupOptions<T>): ExtractionSetup<T>;
+export declare function loadExtractionSetupFromContent<T>(content: string, setupPath: string, options?: LoadExtractionSetupOptions<T>): ExtractionSetup<T>;

package/dist/core/setup.js

@@ -0,0 +1,174 @@
+import { readFileSync } from "node:fs";
+import { dirname, isAbsolute, resolve } from "node:path";
+import { Environment } from "minijinja-js";
+import { getModels, getProviders } from "../models.js";
+import { ExtractError } from "./errors.js";
+import { parseFrontmatter } from "./frontmatter.js";
+export function loadExtractionSetup(setupPath, options = {}) {
+    const resolvedPath = isAbsolute(setupPath) ? setupPath : resolve(process.cwd(), setupPath);
+    const raw = readFileSync(resolvedPath, "utf8");
+    return loadExtractionSetupFromContent(raw, resolvedPath, options);
+}
+export function loadExtractionSetupFromContent(content, setupPath, options = {}) {
+    const resolvedPath = isAbsolute(setupPath) ? setupPath : resolve(process.cwd(), setupPath);
+    const { frontmatter, body } = parseFrontmatter(content);
+    const promptSource = resolvePrompt(frontmatter, body, resolvedPath);
+    const schema = resolveSchema(frontmatter, resolvedPath);
+    const frontmatterOptions = mapFrontmatterToOptions(frontmatter, resolvedPath);
+    const mergedOptions = { ...frontmatterOptions, ...options.overrides, schema, prompt: promptSource };
+    const prompt = renderPromptIfNeeded(promptSource, frontmatter, options.vars, resolvedPath);
+    const finalOptions = { ...mergedOptions, prompt };
+    const model = finalOptions.model;
+    if (!model) {
+        throw new ExtractError(`Missing required "model" in setup file or overrides: ${resolvedPath}`);
+    }
+    const typedOptions = { ...finalOptions, model };
+    return {
+        path: resolvedPath,
+        prompt,
+        frontmatter,
+        options: typedOptions,
+    };
+}
+function resolvePrompt(frontmatter, body, setupPath) {
+    const bodyPrompt = body.trim();
+    if (bodyPrompt.length > 0) {
+        return bodyPrompt;
+    }
+    const frontmatterPrompt = frontmatter.prompt;
+    if (typeof frontmatterPrompt === "string" && frontmatterPrompt.trim().length > 0) {
+        return frontmatterPrompt.trim();
+    }
+    throw new ExtractError(`Missing prompt content in setup file: ${setupPath}`);
+}
+function renderPromptIfNeeded(prompt, frontmatter, vars, setupPath) {
+    const templateEnabled = toBoolean(frontmatter.template);
+    if (!templateEnabled) {
+        return prompt;
+    }
+    const env = new Environment();
+    const mergedVars = { ...resolveFrontmatterVars(frontmatter, setupPath), ...vars };
+    try {
+        return env.renderStr(prompt, mergedVars);
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new ExtractError(`Template rendering failed for ${setupPath}: ${message}`);
+    }
+}
+function resolveSchema(frontmatter, setupPath) {
+    const schemaValue = frontmatter.schema;
+    if (typeof schemaValue !== "string") {
+        throw new ExtractError(`Missing or invalid "schema" in setup file: ${setupPath}`);
+    }
+    const trimmed = schemaValue.trim();
+    if (!trimmed) {
+        throw new ExtractError(`Missing "schema" value in setup file: ${setupPath}`);
+    }
+    if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
+        try {
+            return JSON.parse(trimmed);
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            throw new ExtractError(`Invalid inline schema JSON in setup file ${setupPath}: ${message}`);
+        }
+    }
+    const schemaPath = resolve(dirname(setupPath), trimmed);
+    try {
+        const raw = readFileSync(schemaPath, "utf8");
+        return JSON.parse(raw);
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new ExtractError(`Failed to load schema from ${schemaPath}: ${message}`);
+    }
+}
+function mapFrontmatterToOptions(frontmatter, setupPath) {
+    const modelSpec = frontmatter.model;
+    const model = typeof modelSpec === "string" ? resolveModel(modelSpec, setupPath) : undefined;
+    const apiKey = typeof frontmatter.apiKey === "string" ? frontmatter.apiKey : undefined;
+    const maxTurns = toNumber(frontmatter.maxTurns);
+    const validateCommand = typeof frontmatter.validateCommand === "string" ? frontmatter.validateCommand : undefined;
+    const validateUrl = typeof frontmatter.validateUrl === "string" ? frontmatter.validateUrl : undefined;
+    const cache = normalizeCache(frontmatter.cache);
+    return {
+        model,
+        apiKey,
+        maxTurns,
+        validateCommand,
+        validateUrl,
+        cache,
+    };
+}
+function resolveModel(modelSpec, setupPath) {
+    const [provider, modelId] = modelSpec.split("/");
+    if (!provider || !modelId) {
+        throw new ExtractError(`Invalid model "${modelSpec}" in setup file ${setupPath}. Use "provider/model-id".`);
+    }
+    const providers = getProviders();
+    const providerId = providers.find((item) => item === provider);
+    if (!providerId) {
+        throw new ExtractError(`Unknown provider "${provider}" in setup file ${setupPath}.`);
+    }
+    const model = getModels(providerId).find((candidate) => candidate.id === modelId);
+    if (!model) {
+        throw new ExtractError(`Model "${modelSpec}" not found (from ${setupPath}).`);
+    }
+    return model;
+}
+function resolveFrontmatterVars(frontmatter, setupPath) {
+    const varsValue = frontmatter.vars;
+    if (!varsValue) {
+        return {};
+    }
+    if (typeof varsValue === "object" && !Array.isArray(varsValue)) {
+        return varsValue;
+    }
+    if (typeof varsValue === "string") {
+        const trimmed = varsValue.trim();
+        if (trimmed.startsWith("{") && trimmed.endsWith("}")) {
+            try {
+                const parsed = JSON.parse(trimmed);
+                if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+                    return parsed;
+                }
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                throw new ExtractError(`Invalid vars JSON in setup file ${setupPath}: ${message}`);
+            }
+        }
+    }
+    throw new ExtractError(`Frontmatter "vars" must be an object or JSON object string in ${setupPath}.`);
+}
+function normalizeCache(value) {
+    if (typeof value === "boolean")
+        return value;
+    if (typeof value === "string") {
+        const bool = toBoolean(value);
+        return typeof bool === "boolean" ? bool : undefined;
+    }
+    return undefined;
+}
+function toBoolean(value) {
+    if (typeof value === "boolean")
+        return value;
+    if (typeof value === "string") {
+        const lower = value.trim().toLowerCase();
+        if (lower === "true")
+            return true;
+        if (lower === "false")
+            return false;
+    }
+    return undefined;
+}
+function toNumber(value) {
+    if (typeof value === "number" && Number.isFinite(value)) {
+        return value;
+    }
+    if (typeof value === "string" && /^-?\d+(\.\d+)?$/.test(value.trim())) {
+        return Number(value);
+    }
+    return undefined;
+}

package/dist/core/types.d.ts

@@ -0,0 +1,143 @@
+import type { Api, AssistantMessageEventStream, Context, ImageContent, Model, ThinkingBudgets as PiAiThinkingBudgets, ThinkingLevel as PiAiThinkingLevel } from "@mariozechner/pi-ai";
+import type { TSchema } from "@sinclair/typebox";
+import type { CacheOptions } from "./cache/types.js";
+import type { ExtractEvent } from "./events.js";
+export type StreamFnOptions = {
+    apiKey?: string;
+    signal?: AbortSignal;
+    maxTokens: number;
+    reasoning?: PiAiThinkingLevel;
+    thinkingBudgets?: ThinkingBudgets;
+    onModelSelected?: (model: Model<Api>) => void;
+};
+export type StreamFn = (model: Model<Api>, context: Context, options: StreamFnOptions) => AssistantMessageEventStream;
+/**
+ * Thinking/reasoning level for models that support extended thinking.
+ * - "off": Disable thinking (default)
+ * - "minimal" to "xhigh": Increasing levels of thinking effort
+ *
+ * Note: "xhigh" is only supported by OpenAI gpt-5.1-codex-max, gpt-5.2, and gpt-5.2-codex models.
+ */
+export type ThinkingLevel = "off" | PiAiThinkingLevel;
+/**
+ * Custom token budgets for thinking levels (token-based providers only).
+ */
+export type ThinkingBudgets = PiAiThinkingBudgets;
+/**
+ * Options for extract() and extractSync()
+ */
+export interface ExtractOptions<T> {
+    /**
+     * TypeBox schema defining the expected output structure.
+     * Converted to JSON Schema for the LLM and used by AJV for validation.
+     */
+    schema: TSchema;
+    /**
+     * System prompt instructing the LLM what to extract.
+     * Automatically appended with "You MUST call the 'respond' tool."
+     */
+    prompt: string;
+    /**
+     * LLM model to use. Get one via getModel("provider", "model-id").
+     */
+    model: Model<Api>;
+    /**
+     * API key for the model provider.
+     * @default Reads from environment (ANTHROPIC_API_KEY, OPENAI_API_KEY, etc.)
+     */
+    apiKey?: string;
+    /**
+     * Image attachments for vision-capable models.
+     * Images are sent alongside the text input for multimodal extraction.
+     */
+    attachments?: ImageContent[];
+    /**
+     * Synchronous validation function.
+     * Throw an error to fail validation.
+     */
+    validate?: (data: T) => void;
+    /**
+     * Asynchronous validation function.
+     * Can call external APIs. Throw/reject to fail validation.
+     */
+    validateAsync?: (data: T) => Promise<void>;
+    /**
+     * Shell command for validation.
+     * Receives JSON on stdin. Exit 0 = pass, non-zero = fail (stderr = error).
+     */
+    validateCommand?: string;
+    /**
+     * HTTP endpoint for validation.
+     * POST JSON body. 2xx = pass, 4xx/5xx = fail (body = error).
+     */
+    validateUrl?: string;
+    /**
+     * Maximum turns in the extraction loop. Each turn is one LLM call.
+     * @default 3
+     */
+    maxTurns?: number;
+    /**
+     * Override the LLM stream function (e.g. proxy/routing).
+     */
+    streamFn?: StreamFn;
+    /**
+     * Enable response caching.
+     * @default false
+     */
+    cache?: boolean | CacheOptions;
+    /**
+     * Abort signal for cancellation.
+     */
+    signal?: AbortSignal;
+    /**
+     * Thinking/reasoning level for models that support extended thinking.
+     * Controls how much "thinking effort" the model uses before responding.
+     * @default "off"
+     */
+    thinking?: ThinkingLevel;
+    /**
+     * Custom token budgets for thinking levels (token-based providers only).
+     * Override the default budgets for specific thinking levels.
+     */
+    thinkingBudgets?: ThinkingBudgets;
+}
+/**
+ * Result of a successful extraction.
+ */
+export interface ExtractResult<T> {
+    /** The extracted data, validated against the schema */
+    data: T;
+    /** Number of turns taken (1 = first turn succeeded) */
+    turns: number;
+    /** Cumulative token usage across all turns */
+    usage: Usage;
+}
+/**
+ * Token usage and cost information.
+ */
+export interface Usage {
+    /** Input tokens (prompt + context) */
+    inputTokens: number;
+    /** Output tokens (response) */
+    outputTokens: number;
+    /** Total tokens (input + output) */
+    totalTokens: number;
+    /** Estimated cost in USD */
+    cost: number;
+}
+/**
+ * Async iterable stream of extraction events.
+ */
+export interface ExtractStream<T> extends AsyncIterable<ExtractEvent<T>> {
+    /**
+     * Get the final result.
+     * Resolves when extraction completes.
+     * Returns undefined if extraction failed.
+     */
+    result(): Promise<ExtractResult<T> | undefined>;
+    /**
+     * Abort the extraction.
+     * Causes the stream to emit an error event.
+     */
+    abort(): void;
+}

package/dist/core/types.js

@@ -0,0 +1 @@
+export {};

package/dist/core/validators/assert.d.ts

@@ -0,0 +1 @@
+export declare function runAsserts(data: unknown, asserts: string[], signal?: AbortSignal): Promise<void>;

package/dist/core/validators/assert.js

@@ -0,0 +1,18 @@
+import { AssertionError } from "../errors.js";
+import { runShell } from "./shell.js";
+function shSingleQuote(text) {
+    // Wrap in single quotes, escaping any embedded single quotes:
+    // abc'def -> 'abc'\''def'
+    return `'${text.replaceAll("'", "'\\''")}'`;
+}
+export async function runAsserts(data, asserts, signal) {
+    const json = JSON.stringify(data);
+    for (const expr of asserts) {
+        const result = await runShell(`jq -e ${shSingleQuote(expr)}`, { stdin: json, signal });
+        if (result.code === 0) {
+            continue;
+        }
+        const message = result.stderr.trim() || `Assertion failed: ${expr}`;
+        throw new AssertionError(message, expr);
+    }
+}

package/dist/core/validators/command.d.ts

@@ -0,0 +1 @@
+export declare function runCommandValidator(data: unknown, command: string, signal?: AbortSignal): Promise<void>;

package/dist/core/validators/command.js

@@ -0,0 +1,10 @@
+import { CommandValidationError } from "../errors.js";
+import { runShell } from "./shell.js";
+export async function runCommandValidator(data, command, signal) {
+    const result = await runShell(command, { stdin: JSON.stringify(data), signal });
+    if (result.code === 0) {
+        return;
+    }
+    const message = result.stderr.trim() || `Command exited with code ${result.code}`;
+    throw new CommandValidationError(message, command, result.code);
+}

package/dist/core/validators/http.d.ts

@@ -0,0 +1 @@
+export declare function runHttpValidator(data: unknown, url: string, signal?: AbortSignal): Promise<void>;

package/dist/core/validators/http.js

@@ -0,0 +1,28 @@
+import { HttpValidationError } from "../errors.js";
+async function readErrorBody(response) {
+    const text = await response.text();
+    if (!text) {
+        return "";
+    }
+    try {
+        const json = JSON.parse(text);
+        const msg = typeof json.error === "string" ? json.error : typeof json.message === "string" ? json.message : null;
+        return msg ?? text;
+    }
+    catch {
+        return text;
+    }
+}
+export async function runHttpValidator(data, url, signal) {
+    const response = await fetch(url, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(data),
+        signal,
+    });
+    if (response.ok) {
+        return;
+    }
+    const message = (await readErrorBody(response)) || `Validator returned ${response.status}`;
+    throw new HttpValidationError(message, url, response.status);
+}

package/dist/core/validators/index.d.ts

@@ -0,0 +1,22 @@
+import type { ValidatorLayer } from "../events.js";
+import type { ExtractOptions } from "../types.js";
+export type EmitValidationEvent = (event: {
+    type: "validation_start";
+    layer: ValidatorLayer;
+}) => void;
+export type EmitValidationPassEvent = (event: {
+    type: "validation_pass";
+    layer: ValidatorLayer;
+}) => void;
+export type EmitValidationErrorEvent = (event: {
+    type: "validation_error";
+    layer: ValidatorLayer;
+    error: string;
+}) => void;
+export interface ValidationEmitter {
+    start(layer: ValidatorLayer): void;
+    pass(layer: ValidatorLayer): void;
+    fail(layer: ValidatorLayer, error: string): void;
+}
+export declare function createValidationEmitter(emit: (event: any) => void): ValidationEmitter;
+export declare function runValidators<T>(data: T, options: ExtractOptions<T>, emitter: ValidationEmitter, signal?: AbortSignal): Promise<void>;

package/dist/core/validators/index.js

@@ -0,0 +1,55 @@
+import { runCommandValidator } from "./command.js";
+import { runHttpValidator } from "./http.js";
+export function createValidationEmitter(emit) {
+    return {
+        start: (layer) => emit({ type: "validation_start", layer }),
+        pass: (layer) => emit({ type: "validation_pass", layer }),
+        fail: (layer, error) => emit({ type: "validation_error", layer, error }),
+    };
+}
+export async function runValidators(data, options, emitter, signal) {
+    if (options.validate) {
+        emitter.start("sync");
+        try {
+            options.validate(data);
+            emitter.pass("sync");
+        }
+        catch (e) {
+            emitter.fail("sync", e instanceof Error ? e.message : String(e));
+            throw e;
+        }
+    }
+    if (options.validateAsync) {
+        emitter.start("async");
+        try {
+            await options.validateAsync(data);
+            emitter.pass("async");
+        }
+        catch (e) {
+            emitter.fail("async", e instanceof Error ? e.message : String(e));
+            throw e;
+        }
+    }
+    if (options.validateCommand) {
+        emitter.start("command");
+        try {
+            await runCommandValidator(data, options.validateCommand, signal);
+            emitter.pass("command");
+        }
+        catch (e) {
+            emitter.fail("command", e instanceof Error ? e.message : String(e));
+            throw e;
+        }
+    }
+    if (options.validateUrl) {
+        emitter.start("http");
+        try {
+            await runHttpValidator(data, options.validateUrl, signal);
+            emitter.pass("http");
+        }
+        catch (e) {
+            emitter.fail("http", e instanceof Error ? e.message : String(e));
+            throw e;
+        }
+    }
+}

package/dist/core/validators/shell.d.ts

@@ -0,0 +1,9 @@
+export interface ShellResult {
+    code: number;
+    stdout: string;
+    stderr: string;
+}
+export declare function runShell(command: string, options?: {
+    stdin?: string;
+    signal?: AbortSignal;
+}): Promise<ShellResult>;