@ekairos/thread 1.21.88-beta.0

package/dist/steps/trace.steps.js

@@ -0,0 +1,265 @@
+import "../polyfills/dom-events.js";
+import { lookup } from "@instantdb/admin";
+function requireBaseUrl() {
+    const baseUrl = process.env.EKAIROS_CORE_BASE_URL ||
+        process.env.EKAIROS_TRACES_BASE_URL ||
+        process.env.EKAIROS_BASE_URL;
+    if (!baseUrl) {
+        throw new Error("[thread/trace] Missing EKAIROS_CORE_BASE_URL (or EKAIROS_TRACES_BASE_URL)");
+    }
+    return baseUrl.replace(/\/$/, "");
+}
+function requireToken() {
+    // Preferred: Clerk org API key (opaque token) for ekairos-core.
+    const apiKey = process.env.EKAIROS_CLERK_API_KEY;
+    if (apiKey)
+        return apiKey;
+    throw new Error("[thread/trace] Missing EKAIROS_CLERK_API_KEY");
+}
+let jwtCache = null;
+function parseJwtExpMs(token) {
+    const parts = token.split(".");
+    if (parts.length !== 3)
+        return null;
+    try {
+        const payload = JSON.parse(Buffer.from(parts[1].replace(/-/g, "+").replace(/_/g, "/"), "base64").toString("utf-8"));
+        const exp = typeof payload?.exp === "number" ? payload.exp : null;
+        return exp ? exp * 1000 : null;
+    }
+    catch {
+        return null;
+    }
+}
+async function getTraceAuthHeader(baseUrl, projectId) {
+    const apiKey = requireToken();
+    const now = Date.now();
+    if (jwtCache && jwtCache.expMs - 60000 > now) {
+        return `Bearer ${jwtCache.token}`;
+    }
+    try {
+        const res = await fetch(`${baseUrl}/api/thread/traces/auth`, {
+            method: "POST",
+            headers: {
+                "content-type": "application/json",
+                authorization: `Bearer ${apiKey}`,
+            },
+            body: JSON.stringify({ projectId }),
+        });
+        if (res.ok) {
+            const json = (await res.json());
+            const token = typeof json?.token === "string" ? json.token : "";
+            const expMs = parseJwtExpMs(token) ?? now + 60 * 60 * 1000;
+            if (token) {
+                jwtCache = { token, expMs };
+                return `Bearer ${token}`;
+            }
+        }
+    }
+    catch {
+        // fall back to API key below
+    }
+    return `Bearer ${apiKey}`;
+}
+async function readProjectId() {
+    const { getRuntimeProjectId } = await import("@ekairos/domain/runtime");
+    const fromConfig = String(getRuntimeProjectId() || "").trim();
+    if (fromConfig)
+        return fromConfig;
+    const fallback = typeof process !== "undefined" && process.env
+        ? String(process.env.EKAIROS_PROJECT_ID || "").trim()
+        : "";
+    return fallback;
+}
+export async function writeThreadTraceEvents(params) {
+    if (!params.events?.length)
+        return;
+    const envTrace = params.env?.traces;
+    // Tracing must NEVER break workflows by default.
+    // Use EKAIROS_TRACES_STRICT=1 if you want to fail hard.
+    const strict = envTrace?.strict === true || process.env.EKAIROS_TRACES_STRICT === "1";
+    // 1) Local trace persistence (InstantDB source of truth).
+    try {
+        const { getThreadRuntime } = await import("@ekairos/thread/runtime");
+        const runtime = await getThreadRuntime(params.env);
+        const db = runtime?.db;
+        if (db) {
+            const now = new Date();
+            const orgId = typeof params.env?.orgId === "string"
+                ? String(params.env.orgId)
+                : "";
+            const projectId = await readProjectId();
+            const byRun = new Map();
+            for (const ev of params.events) {
+                const runId = String(ev.workflowRunId || "");
+                if (!runId)
+                    continue;
+                if (!byRun.has(runId))
+                    byRun.set(runId, []);
+                byRun.get(runId).push(ev);
+            }
+            const seqByRun = new Map();
+            const existingCountByRun = new Map();
+            for (const [runId] of byRun) {
+                let existingCount = 0;
+                try {
+                    const q = await db.query({
+                        thread_trace_runs: {
+                            $: { where: { workflowRunId: runId }, limit: 1 },
+                        },
+                    });
+                    const row = q?.thread_trace_runs?.[0];
+                    existingCount = Number(row?.eventsCount ?? 0) || 0;
+                }
+                catch {
+                    // ignore
+                }
+                existingCountByRun.set(runId, existingCount);
+                seqByRun.set(runId, existingCount);
+            }
+            const txs = [];
+            const spanTxs = [];
+            for (const ev of params.events) {
+                const runId = String(ev.workflowRunId || "");
+                if (!runId)
+                    continue;
+                const key = `${runId}:${String(ev.eventId || "")}`;
+                if (!key.includes(":"))
+                    continue;
+                const eventAt = typeof ev.eventAt === "string" && ev.eventAt
+                    ? new Date(ev.eventAt)
+                    : undefined;
+                let seq = Number.isFinite(Number(ev.seq)) ? Number(ev.seq) : undefined;
+                if (typeof seq !== "number") {
+                    const current = seqByRun.get(runId) ?? 0;
+                    const next = current + 1;
+                    seqByRun.set(runId, next);
+                    seq = next;
+                }
+                ev.seq = seq;
+                txs.push(db.tx.thread_trace_events[lookup("key", key)].update({
+                    key,
+                    workflowRunId: runId,
+                    seq,
+                    eventId: String(ev.eventId || ""),
+                    eventKind: String(ev.eventKind || ""),
+                    eventAt: eventAt ?? undefined,
+                    ingestedAt: now,
+                    orgId: orgId || undefined,
+                    projectId: projectId || undefined,
+                    contextKey: ev.contextKey,
+                    spanId: ev.spanId,
+                    parentSpanId: ev.parentSpanId,
+                    contextId: ev.contextId,
+                    executionId: ev.executionId,
+                    stepId: ev.stepId,
+                    contextEventId: ev.contextEventId,
+                    toolCallId: ev.toolCallId,
+                    partKey: ev.partKey,
+                    partIdx: ev.partIdx,
+                    isDeleted: ev.isDeleted === true,
+                    aiProvider: ev.aiProvider,
+                    aiModel: ev.aiModel,
+                    promptTokens: ev.promptTokens,
+                    promptTokensCached: ev.promptTokensCached,
+                    promptTokensUncached: ev.promptTokensUncached,
+                    completionTokens: ev.completionTokens,
+                    totalTokens: ev.totalTokens,
+                    latencyMs: ev.latencyMs,
+                    cacheCostUsd: ev.cacheCostUsd,
+                    computeCostUsd: ev.computeCostUsd,
+                    costUsd: ev.costUsd,
+                    payload: ev.payload,
+                }));
+                if (ev.eventKind === "thread.step" || ev.eventKind === "workflow.step") {
+                    const spanId = String(ev.stepId || ev.eventId || key);
+                    spanTxs.push(db.tx.thread_trace_spans[lookup("spanId", spanId)].update({
+                        spanId,
+                        parentSpanId: ev.parentSpanId,
+                        workflowRunId: runId,
+                        executionId: ev.executionId,
+                        stepId: ev.stepId,
+                        kind: ev.eventKind,
+                        name: ev.eventKind,
+                        status: "completed",
+                        startedAt: eventAt ?? now,
+                        endedAt: eventAt ?? now,
+                        durationMs: 0,
+                        payload: ev.payload,
+                    }));
+                }
+            }
+            if (txs.length) {
+                await db.transact(txs);
+            }
+            if (spanTxs.length) {
+                await db.transact(spanTxs);
+            }
+            for (const [runId, events] of byRun) {
+                const eventDates = events
+                    .map((e) => typeof e.eventAt === "string" && e.eventAt
+                    ? new Date(e.eventAt)
+                    : now)
+                    .filter((d) => !Number.isNaN(d.getTime()));
+                const firstEventAt = eventDates.length
+                    ? new Date(Math.min(...eventDates.map((d) => d.getTime())))
+                    : now;
+                const lastEventAt = eventDates.length
+                    ? new Date(Math.max(...eventDates.map((d) => d.getTime())))
+                    : now;
+                const existingCount = existingCountByRun.get(runId) ?? 0;
+                await db.transact([
+                    db.tx.thread_trace_runs[lookup("workflowRunId", runId)].update({
+                        workflowRunId: runId,
+                        orgId: orgId || undefined,
+                        projectId: projectId || undefined,
+                        firstEventAt,
+                        lastEventAt,
+                        lastIngestedAt: now,
+                        eventsCount: existingCount + events.length,
+                    }),
+                ]);
+            }
+        }
+    }
+    catch (e) {
+        if (strict)
+            throw e;
+    }
+    let baseUrl = "";
+    try {
+        baseUrl = envTrace?.baseUrl ? String(envTrace.baseUrl).replace(/\/$/, "") : requireBaseUrl();
+    }
+    catch (e) {
+        if (strict)
+            throw e;
+        return;
+    }
+    const projectId = envTrace?.projectId ? String(envTrace.projectId).trim() : await readProjectId();
+    if (!projectId) {
+        if (strict)
+            throw new Error("[thread/trace] Missing projectId (ekairosConfig or EKAIROS_PROJECT_ID)");
+        return;
+    }
+    const authHeader = envTrace?.apiKey
+        ? `Bearer ${String(envTrace.apiKey).trim()}`
+        : await getTraceAuthHeader(baseUrl, projectId);
+    const res = await fetch(`${baseUrl}/api/thread/traces/ingest`, {
+        method: "POST",
+        headers: {
+            "content-type": "application/json",
+            authorization: authHeader,
+        },
+        body: JSON.stringify({ projectId, events: params.events }),
+    });
+    if (!res.ok) {
+        const text = await res.text().catch(() => "");
+        if (strict) {
+            throw new Error(`[thread/trace] ekairos-core ingest failed (${res.status}): ${text}`);
+        }
+        if (process.env.PLAYWRIGHT_TEST === "1") {
+            // eslint-disable-next-line no-console
+            console.warn(`[thread/trace] ingest failed (${res.status}): ${text}`);
+        }
+        return;
+    }
+}

package/dist/stores/instant.document-parser.d.ts

@@ -0,0 +1,6 @@
+import "../polyfills/dom-events.js";
+/**
+ * Parses a document with LlamaParse and stores it in InstantDB (document_documents + link to file).
+ * Returns the created documentId.
+ */
+export declare function parseAndStoreDocument(db: any, buffer: Buffer, fileName: string, fileId: string): Promise<string>;

package/dist/stores/instant.document-parser.js

@@ -0,0 +1,210 @@
+import "../polyfills/dom-events.js";
+import { id } from "@instantdb/admin";
+const LLAMA_CLOUD_BASE_URL = "https://api.cloud.llamaindex.ai/api/v1";
+function safeErrorJson(error) {
+    const seen = new WeakSet();
+    const redactKey = (k) => /token|authorization|cookie|secret|api[_-]?key|password/i.test(k);
+    const err = error;
+    const payload = {
+        name: err?.name,
+        message: err?.message,
+        status: err?.status,
+        body: err?.body,
+        data: err?.data,
+        stack: err?.stack,
+    };
+    try {
+        return JSON.stringify(payload, (k, v) => {
+            if (redactKey(k))
+                return "[redacted]";
+            if (typeof v === "string" && v.length > 5000)
+                return "[truncated-string]";
+            if (typeof v === "object" && v !== null) {
+                if (seen.has(v))
+                    return "[circular]";
+                seen.add(v);
+            }
+            return v;
+        });
+    }
+    catch {
+        return JSON.stringify({ message: String(err?.message ?? "error") });
+    }
+}
+async function probeInstantDocumentSchema(db) {
+    // Best-effort probes to pinpoint missing schema pieces WITHOUT logging dynamic payloads.
+    // Each line is a static string.
+    try {
+        await db.query({ document_documents: { $: { limit: 1 } } });
+    }
+    catch {
+        console.error("Instant schema probe: document_documents entity query failed");
+    }
+    try {
+        await db.query({ document_documents: { $: { limit: 1 }, file: {} } });
+    }
+    catch {
+        console.error("Instant schema probe: document_documents.file link query failed");
+    }
+    try {
+        await db.query({ $files: { $: { limit: 1 } } });
+    }
+    catch {
+        console.error("Instant schema probe: $files entity query failed");
+    }
+    try {
+        await db.query({ $files: { $: { limit: 1 }, document: {} } });
+    }
+    catch {
+        console.error("Instant schema probe: $files.document link query failed");
+    }
+}
+async function uploadToLlamaCloud(buffer, fileName) {
+    const formData = new FormData();
+    const uint8Array = new Uint8Array(buffer);
+    const blob = new Blob([uint8Array], { type: "application/pdf" });
+    formData.append("file", blob, fileName);
+    formData.append("parse_mode", "parse_page_with_llm");
+    formData.append("high_res_ocr", "true");
+    formData.append("adaptive_long_table", "true");
+    formData.append("outlined_table_extraction", "true");
+    formData.append("output_tables_as_HTML", "true");
+    let response;
+    try {
+        response = await fetch(`${LLAMA_CLOUD_BASE_URL}/parsing/upload`, {
+            method: "POST",
+            headers: {
+                Authorization: `Bearer ${process.env.LLAMA_CLOUD_API_KEY}`,
+            },
+            body: formData,
+        });
+    }
+    catch (error) {
+        console.error("LlamaCloud: upload fetch threw", safeErrorJson(error));
+        throw error;
+    }
+    if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(`LlamaCloud upload failed: ${response.status} ${errorText}`);
+    }
+    const result = (await response.json());
+    return result.id;
+}
+async function getJobStatus(jobId) {
+    let response;
+    try {
+        response = await fetch(`${LLAMA_CLOUD_BASE_URL}/parsing/job/${jobId}`, {
+            method: "GET",
+            headers: {
+                Authorization: `Bearer ${process.env.LLAMA_CLOUD_API_KEY}`,
+            },
+        });
+    }
+    catch (error) {
+        console.error("LlamaCloud: status fetch threw", safeErrorJson(error));
+        throw error;
+    }
+    if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(`LlamaCloud status fetch failed: ${response.status} ${errorText}`);
+    }
+    return (await response.json());
+}
+async function getParseResult(jobId) {
+    let response;
+    try {
+        response = await fetch(`${LLAMA_CLOUD_BASE_URL}/parsing/job/${jobId}/result/markdown`, {
+            method: "GET",
+            headers: {
+                Authorization: `Bearer ${process.env.LLAMA_CLOUD_API_KEY}`,
+            },
+        });
+    }
+    catch (error) {
+        console.error("LlamaCloud: result fetch threw", safeErrorJson(error));
+        throw error;
+    }
+    if (!response.ok) {
+        const errorText = await response.text();
+        throw new Error(`LlamaCloud result fetch failed: ${response.status} ${errorText}`);
+    }
+    return (await response.json());
+}
+async function waitForProcessing(jobId, maxAttempts = 60) {
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+        const statusResponse = await getJobStatus(jobId);
+        if (statusResponse.status === "SUCCESS" || statusResponse.status === "COMPLETED") {
+            return await getParseResult(jobId);
+        }
+        if (statusResponse.status === "ERROR" || statusResponse.status === "FAILED") {
+            throw new Error(`LlamaCloud processing failed with status: ${statusResponse.status}`);
+        }
+        await new Promise((resolve) => setTimeout(resolve, 2000));
+    }
+    throw new Error("LlamaCloud processing timeout");
+}
+/**
+ * Parses a document with LlamaParse and stores it in InstantDB (document_documents + link to file).
+ * Returns the created documentId.
+ */
+export async function parseAndStoreDocument(db, buffer, fileName, fileId) {
+    let existingDocument;
+    try {
+        existingDocument = await db.query({
+            document_documents: {
+                $: {
+                    where: { "file.id": fileId },
+                },
+                file: {},
+            },
+        });
+    }
+    catch (error) {
+        console.error("parseAndStoreDocument: query existing failed", safeErrorJson(error));
+        throw error;
+    }
+    if (existingDocument.document_documents && existingDocument.document_documents.length > 0) {
+        return existingDocument.document_documents[0].id;
+    }
+    const jobId = await uploadToLlamaCloud(buffer, fileName);
+    const result = await waitForProcessing(jobId);
+    const pages = [];
+    if (result.markdown) {
+        pages.push({
+            id: id(),
+            text: result.markdown,
+        });
+    }
+    if (result.pages && result.pages.length > 0) {
+        for (const page of result.pages) {
+            pages.push({
+                id: id(),
+                text: page.text,
+            });
+        }
+    }
+    if (pages.length === 0) {
+        throw new Error("No content extracted from document");
+    }
+    const documentId = id();
+    try {
+        await db.transact([
+            db.tx.document_documents[documentId].update({
+                content: { pages },
+                name: fileName,
+                mimeType: "application/pdf",
+                createdAt: new Date(),
+            }),
+            db.tx.document_documents[documentId].link({
+                file: fileId,
+            }),
+        ]);
+    }
+    catch (error) {
+        console.error("parseAndStoreDocument: transact failed", safeErrorJson(error));
+        // Diagnose missing schema attributes/links (static logs only).
+        await probeInstantDocumentSchema(db);
+        throw error;
+    }
+    return documentId;
+}

package/dist/stores/instant.documents.d.ts

@@ -0,0 +1,16 @@
+import type { ThreadItem } from "../thread.store.js";
+export declare function coerceDocumentTextPages(documentRecord: any, opts?: {
+    pageLabelPrefix?: string;
+}): string;
+export declare function expandEventsWithInstantDocuments(params: {
+    db: any;
+    events: ThreadItem[];
+    /**
+     * Hard limit to avoid huge model inputs. Defaults to 120k chars of extracted text.
+     */
+    maxChars?: number;
+    /**
+     * Event type used for derived document text. Defaults to "document.parsed".
+     */
+    derivedEventType?: string;
+}): Promise<ThreadItem[]>;

package/dist/stores/instant.documents.js

@@ -0,0 +1,152 @@
+import { parseAndStoreDocument } from "./instant.document-parser.js";
+function isFilePart(part) {
+    return Boolean(part &&
+        typeof part === "object" &&
+        (part.type === "file" || part?.providerMetadata?.instant));
+}
+function formatAttachmentSummary(part) {
+    const instant = part?.providerMetadata?.instant ?? {};
+    const fileId = typeof instant?.fileId === "string" ? instant.fileId : "";
+    const filename = typeof part?.filename === "string" ? part.filename : "";
+    const mediaType = typeof part?.mediaType === "string" ? part.mediaType : "";
+    // Keep it compact; no URLs (can be signed/sensitive).
+    return `fileId="${fileId}" filename="${filename}" mediaType="${mediaType}"`;
+}
+export function coerceDocumentTextPages(documentRecord, opts) {
+    const pages = documentRecord?.content?.pages;
+    if (!Array.isArray(pages) || pages.length === 0)
+        return "";
+    const prefix = opts?.pageLabelPrefix ?? "Page";
+    return pages
+        .map((p, idx) => {
+        const text = typeof p?.text === "string" ? p.text : "";
+        return `\n\n--- ${prefix} ${idx + 1} ---\n\n${text}`;
+    })
+        .join("");
+}
+async function resolveInstantFileRecord(db, params) {
+    const fileId = params.fileId ? String(params.fileId) : null;
+    const filePath = params.path ? String(params.path) : null;
+    if (!fileId && !filePath)
+        return null;
+    if (fileId) {
+        const q = await db.query({
+            $files: { $: { where: { id: fileId }, limit: 1 }, document: {} },
+        });
+        return q?.$files?.[0] ?? null;
+    }
+    const q = await db.query({
+        $files: { $: { where: { path: filePath }, limit: 1 }, document: {} },
+    });
+    return q?.$files?.[0] ?? null;
+}
+async function ensureDocumentParsedForFile(db, params) {
+    const fileRecord = params.fileRecord;
+    const part = params.part;
+    let documentRecord = Array.isArray(fileRecord?.document)
+        ? fileRecord.document?.[0]
+        : fileRecord.document;
+    if (documentRecord?.id)
+        return documentRecord;
+    const fileUrl = typeof fileRecord?.url === "string" ? fileRecord.url : "";
+    if (!fileUrl.startsWith("http://") && !fileUrl.startsWith("https://")) {
+        return null;
+    }
+    const resp = await fetch(fileUrl);
+    if (!resp.ok)
+        throw new Error(`Failed to fetch file for parsing: HTTP ${resp.status}`);
+    const buffer = Buffer.from(await resp.arrayBuffer());
+    const name = (typeof part?.filename === "string" && part.filename) ||
+        (typeof fileRecord?.path === "string" && fileRecord.path) ||
+        "file";
+    // NOTE: Do not invent fallback paths. If the file doesn't have a stable `path`,
+    // we don't fabricate one.
+    const path = typeof fileRecord?.path === "string" ? fileRecord.path : undefined;
+    const documentId = await parseAndStoreDocument(db, buffer, name, String(fileRecord.id));
+    const dq = await db.query({
+        document_documents: { $: { where: { id: documentId }, limit: 1 }, file: {} },
+    });
+    documentRecord = dq?.document_documents?.[0] ?? null;
+    return documentRecord;
+}
+export async function expandEventsWithInstantDocuments(params) {
+    const db = params.db;
+    const maxChars = typeof params.maxChars === "number" ? params.maxChars : 120000;
+    const derivedEventType = params.derivedEventType ?? "document.parsed";
+    const out = [];
+    for (const event of params.events) {
+        const parts = event?.content?.parts;
+        if (!Array.isArray(parts) || parts.length === 0) {
+            out.push(event);
+            continue;
+        }
+        const hadFileParts = parts.some((p) => isFilePart(p));
+        if (hadFileParts) {
+            // Do not forward file parts to the model (gateways may not support some media types).
+            // The derived `document.parsed` event contains the extracted text.
+            const filtered = parts.filter((p) => !isFilePart(p));
+            const attachmentSummaries = parts
+                .filter((p) => isFilePart(p))
+                .map((p) => formatAttachmentSummary(p))
+                .join("\n");
+            const attachmentInfoText = attachmentSummaries
+                ? `Attachment info:\n${attachmentSummaries}`
+                : "Attachment info: (unavailable)";
+            const sanitized = {
+                ...event,
+                content: {
+                    ...event?.content,
+                    parts: [
+                        ...filtered,
+                        {
+                            type: "text",
+                            text: "[Attachment omitted from model input. Parsed content will follow in a document.parsed event.]\n" +
+                                attachmentInfoText,
+                        },
+                    ],
+                },
+            };
+            out.push(sanitized);
+        }
+        else {
+            out.push(event);
+        }
+        for (const part of parts) {
+            if (!isFilePart(part))
+                continue;
+            const instantMeta = part?.providerMetadata?.instant ?? {};
+            const fileId = instantMeta?.fileId ? String(instantMeta.fileId) : undefined;
+            const filePath = instantMeta?.path ? String(instantMeta.path) : undefined;
+            const fileRecord = await resolveInstantFileRecord(db, { fileId, path: filePath });
+            if (!fileRecord?.id)
+                continue;
+            const documentRecord = await ensureDocumentParsedForFile(db, { fileRecord, part });
+            const pageText = coerceDocumentTextPages(documentRecord);
+            if (!pageText)
+                continue;
+            const clipped = pageText.length > maxChars
+                ? `${pageText.slice(0, maxChars)}\n\n[truncated: maxChars=${maxChars}]`
+                : pageText;
+            const derivedAttachmentInfo = `Attachment info:\n${formatAttachmentSummary(part)}`;
+            const derived = {
+                id: `derived:${event.id}:${String(fileRecord.id)}`,
+                type: derivedEventType,
+                channel: event.channel ?? "web",
+                createdAt: new Date().toISOString(),
+                content: {
+                    parts: [
+                        {
+                            type: "text",
+                            text: "Parsed document available.\n" +
+                                derivedAttachmentInfo +
+                                "\nProvider: llamacloud",
+                        },
+                        { type: "text", text: `Document transcription:${clipped}` },
+                    ],
+                },
+            };
+            out.push(derived);
+        }
+    }
+    return out;
+}