@promptev/context-engine 0.0.1 → 0.0.2

package/README.md

@@ -147,6 +147,29 @@ trusted caller and returns the entire corpus. Use `[]` or `TRUSTED` explicitly.
 npx context-engine install-skill
 ```
 
+## Compute over tables
+
+`engine.compute()` turns the in-scope CSV/XLSX documents into row tables and has
+the LLM write code against them. `computeOverFrames()` is the same path for
+tables you already hold — an uploaded workbook, a connector's sheet, a query
+result — with no document to point at:
+
+```ts
+import { computeOverFrames } from "@promptev/context-engine";
+
+const out = await computeOverFrames(
+  { expenses: rows },             // { sheetName: Array<Record<string, unknown>> }
+  "total the amount column",
+  { config: engine.config },
+);
+out.result;
+```
+
+Both are **off by default** (`enableCodeExecution: true` turns them on — only
+behind real OS-level isolation) and both apply `config.redaction` before the LLM
+sees anything: every string cell *and* every column header is masked, and the
+returned object — generated code included — is swept on the way out.
+
 ## Serve
 
 ```ts

package/dist/cli.js

@@ -414,17 +414,30 @@ var init_config = __esm({
   "src/config.ts"() {
     init_redaction();
     embeddingSchema = z.object({
-      provider: z.enum(["openai", "azure_openai", "gemini", "voyage", "cohere", "custom"]),
+      provider: z.enum(["openai", "azure_openai", "gemini", "vertex_ai", "voyage", "cohere", "custom"]),
       model: z.string(),
       dim: z.number().int().positive().nullable().optional().default(null),
       apiKey: z.string().nullable().optional().default(null),
-      baseUrl: z.string().nullable().optional().default(null)
+      baseUrl: z.string().nullable().optional().default(null),
+      // `vertex_ai` only. Left optional on purpose: the Google SDK resolves both
+      // from GOOGLE_CLOUD_PROJECT / GOOGLE_CLOUD_LOCATION, which is how a GCP
+      // deployment is already wired, and requiring them here would break it.
+      //
+      // No `.default(null)`, unlike the fields above — these types come from
+      // `z.infer`, so a default would make them REQUIRED on the output type and
+      // break every hand-written `EmbeddingConfig` literal already compiled
+      // against 0.0.1. Optional keeps the addition additive.
+      project: z.string().nullable().optional(),
+      location: z.string().nullable().optional()
     });
     llmSchema = z.object({
-      provider: z.enum(["anthropic", "openai", "azure_openai", "gemini", "bedrock", "custom"]),
+      provider: z.enum(["anthropic", "openai", "azure_openai", "gemini", "vertex_ai", "bedrock", "custom"]),
       model: z.string(),
       apiKey: z.string().nullable().optional().default(null),
-      baseUrl: z.string().nullable().optional().default(null)
+      baseUrl: z.string().nullable().optional().default(null),
+      // `vertex_ai` only — see the note on embeddingSchema.
+      project: z.string().nullable().optional(),
+      location: z.string().nullable().optional()
     });
     graphSchema = z.object({
       enabled: z.boolean().default(false),
@@ -2267,6 +2280,47 @@ var init_chunkers = __esm({
   }
 });
 
+// src/providers/google.ts
+async function buildGenaiClient(cfg2, timeoutMs, purpose) {
+  const specifier = "@google/genai";
+  let mod;
+  try {
+    mod = await import(specifier);
+  } catch {
+    throw new ExtraMissingError("gemini", specifier, purpose);
+  }
+  const Ctor = mod.GoogleGenAI ?? mod.Client;
+  if (!Ctor) {
+    throw new ExtraMissingError("gemini", specifier, purpose);
+  }
+  const opts = { httpOptions: { timeout: timeoutMs } };
+  if (cfg2.provider === "vertex_ai") {
+    opts.vertexai = true;
+    if (cfg2.project || cfg2.location) {
+      if (cfg2.project) opts.project = cfg2.project;
+      if (cfg2.location) opts.location = cfg2.location;
+    } else if (cfg2.apiKey) {
+      opts.apiKey = cfg2.apiKey;
+    }
+  } else {
+    opts.apiKey = cfg2.apiKey ?? null;
+  }
+  try {
+    return new Ctor(opts);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    if (!message.includes("Authentication is not set up")) throw err;
+    throw new Error(
+      `${message} Set \`project\` on the provider config, or export GOOGLE_CLOUD_PROJECT and GOOGLE_CLOUD_LOCATION. Credentials themselves come from Application Default Credentials.`
+    );
+  }
+}
+var init_google = __esm({
+  "src/providers/google.ts"() {
+    init_errors();
+  }
+});
+
 // src/providers/llm.ts
 var llm_exports = {};
 __export(llm_exports, {
@@ -2302,20 +2356,6 @@ function buildOpenAIChatClient(cfg2) {
     maxRetries: 0
   });
 }
-async function loadGeminiChatClient(apiKey) {
-  const specifier = "@google/genai";
-  let mod;
-  try {
-    mod = await import(specifier);
-  } catch {
-    throw new ExtraMissingError("gemini", specifier, "gemini llm");
-  }
-  const Ctor = mod.GoogleGenAI ?? mod.Client;
-  if (!Ctor) {
-    throw new ExtraMissingError("gemini", specifier, "gemini llm");
-  }
-  return new Ctor({ apiKey: apiKey ?? null, httpOptions: { timeout: GEMINI_CALL_TIMEOUT_MS } });
-}
 async function loadBedrockSdk() {
   const specifier = "@aws-sdk/client-bedrock-runtime";
   try {
@@ -2334,7 +2374,7 @@ function buildLlmClient(cfg2, opts) {
   if (OPENAI_FAMILY.has(cfg2.provider)) {
     return new LLMClient(cfg2, { client: buildOpenAIChatClient(cfg2) });
   }
-  if (cfg2.provider === "gemini" || cfg2.provider === "bedrock") {
+  if (GOOGLE_FAMILY.has(cfg2.provider) || cfg2.provider === "bedrock") {
     return new LLMClient(cfg2);
   }
   throw new Error(`unknown llm provider: ${JSON.stringify(cfg2.provider)}`);
@@ -2351,16 +2391,18 @@ async function callLlm(cfg2, opts) {
     await owned.aclose();
   }
 }
-var CALL_TIMEOUT_MS, TIMEOUT_MS, GEMINI_CALL_TIMEOUT_MS, ANTHROPIC_VERSION, ANTHROPIC_MAX_TOKENS, OPENAI_FAMILY, LLMClient;
+var CALL_TIMEOUT_MS, TIMEOUT_MS, GEMINI_CALL_TIMEOUT_MS, ANTHROPIC_VERSION, ANTHROPIC_MAX_TOKENS, OPENAI_FAMILY, GOOGLE_FAMILY, LLMClient;
 var init_llm = __esm({
   "src/providers/llm.ts"() {
     init_errors();
+    init_google();
     CALL_TIMEOUT_MS = 24e4;
     TIMEOUT_MS = CALL_TIMEOUT_MS;
     GEMINI_CALL_TIMEOUT_MS = CALL_TIMEOUT_MS;
     ANTHROPIC_VERSION = "2023-06-01";
     ANTHROPIC_MAX_TOKENS = 4096;
     OPENAI_FAMILY = /* @__PURE__ */ new Set(["openai", "azure_openai", "custom"]);
+    GOOGLE_FAMILY = /* @__PURE__ */ new Set(["gemini", "vertex_ai"]);
     LLMClient = class {
       cfg;
       provider;
@@ -2400,7 +2442,7 @@ var init_llm = __esm({
         if (OPENAI_FAMILY.has(this.provider)) {
           return this.callOpenAI(system, user, jsonMode, images, maxTokens, temperature);
         }
-        if (this.provider === "gemini") {
+        if (GOOGLE_FAMILY.has(this.provider)) {
           return this.callGemini(system, user, jsonMode, images, maxTokens, thinkingBudget, temperature);
         }
         if (this.provider === "bedrock") {
@@ -2483,7 +2525,11 @@ Respond with valid JSON only.`;
       }
       async callGemini(system, user, jsonMode, images, maxTokens, thinkingBudget = null, temperature = null) {
         if (!this.genaiClient) {
-          this.genaiClient = await loadGeminiChatClient(this.cfg.apiKey);
+          this.genaiClient = await buildGenaiClient(
+            this.cfg,
+            GEMINI_CALL_TIMEOUT_MS,
+            "gemini llm"
+          );
         }
         const parts = [];
         for (const img of images ?? []) {
@@ -6399,20 +6445,8 @@ function parseSpreadsheetText(dfd, text) {
   return out;
 }
 async function compute(instruction, opts) {
-  if (!opts.config.enableCodeExecution) {
-    throw new EngineActionError(
-      "compute() executes generated code and is disabled by default; set enableCodeExecution=true only in a deployment with out-of-process/container isolation."
-    );
-  }
-  const llmCfg = opts.modelCfg ?? opts.config.llm;
-  if (llmCfg == null) {
-    throw new Error("compute() requires an LLM: pass modelCfg= or configure ContextEngineConfig.llm");
-  }
-  if (!instruction?.trim()) {
-    throw new EngineActionError("instruction must not be empty");
-  }
+  checkComputePreconditions(opts.config, opts.modelCfg, instruction);
   const dfd = await requireDanfo();
-  const timeout = Math.max(1, Math.min(Math.trunc(opts.timeout || DEFAULT_COMPUTE_TIMEOUT), 300));
   const principals = opts.principals ?? null;
   const params = [];
   let where = scopeSql(opts.sourceIds ?? null, principals, params);
@@ -6482,6 +6516,69 @@ async function compute(instruction, opts) {
   if (!Object.keys(dfs).length) {
     throw new EngineActionError("in-scope documents did not parse into any usable dataframe");
   }
+  return computeOverFrames(dfs, instruction, {
+    config: opts.config,
+    modelCfg: opts.modelCfg,
+    timeout: opts.timeout,
+    hooks: opts.hooks,
+    principals,
+    documents
+  });
+}
+function checkComputePreconditions(config, modelCfg, instruction) {
+  if (!config.enableCodeExecution) {
+    throw new EngineActionError(
+      "compute() executes generated code and is disabled by default; set enableCodeExecution=true only in a deployment with out-of-process/container isolation."
+    );
+  }
+  const llmCfg = modelCfg ?? config.llm;
+  if (llmCfg == null) {
+    throw new Error("compute() requires an LLM: pass modelCfg= or configure ContextEngineConfig.llm");
+  }
+  if (!instruction?.trim()) {
+    throw new EngineActionError("instruction must not be empty");
+  }
+  return llmCfg;
+}
+function maskFrames(frames, policy, opts) {
+  if (policy == null || policy.isEmpty()) return frames;
+  const mask = (text) => redactValueRecursive(text, policy, opts);
+  const masked = {};
+  for (const [name, rows] of Object.entries(frames)) {
+    const labels = /* @__PURE__ */ new Map();
+    const taken = /* @__PURE__ */ new Set();
+    for (const row of rows) {
+      for (const column of Object.keys(row)) {
+        if (labels.has(column)) continue;
+        const base = mask(column);
+        let label = base;
+        for (let n = 2; taken.has(label); n++) label = `${base}_${n}`;
+        taken.add(label);
+        labels.set(column, label);
+      }
+    }
+    masked[name] = rows.map(
+      (row) => Object.fromEntries(
+        Object.entries(row).map(([column, value]) => [
+          labels.get(column) ?? column,
+          typeof value === "string" ? mask(value) : value
+        ])
+      )
+    );
+  }
+  return masked;
+}
+async function computeOverFrames(frames, instruction, opts) {
+  const llmCfg = checkComputePreconditions(opts.config, opts.modelCfg, instruction);
+  if (!frames || !Object.keys(frames).length) {
+    throw new EngineActionError("no tabular data to compute over");
+  }
+  const hooks = opts.hooks ?? {};
+  const principals = opts.principals ?? null;
+  const documents = [...opts.documents ?? []];
+  const timeout = Math.max(1, Math.min(Math.trunc(opts.timeout || DEFAULT_COMPUTE_TIMEOUT), 300));
+  const redactOpts = { principals, secretKey: opts.config.secretKey, hooks };
+  const dfs = maskFrames(frames, opts.config.redaction, redactOpts);
   const schemaLines = Object.entries(dfs).map(
     ([name, table]) => `- ${name}: columns=${JSON.stringify(Object.keys(table[0] ?? {}))}, rows=${table.length}`
   );
@@ -6498,7 +6595,7 @@ ${schemaLines.join("\n")}`;
       jsonMode: false
     });
   } catch (exc) {
-    emitError(opts.hooks, exc, { stage: "compute_codegen", instruction: instruction.slice(0, 200) });
+    emitError(hooks, exc, { stage: "compute_codegen", instruction: instruction.slice(0, 200) });
     throw exc;
   }
   const code = stripCodeFences(rawCode);
@@ -6509,12 +6606,12 @@ ${schemaLines.join("\n")}`;
       maskedCode = redactValueRecursive(code.slice(0, 500), opts.config.redaction, {
         principals,
         secretKey: opts.config.secretKey,
-        hooks: opts.hooks
+        hooks
       });
     } catch {
       maskedCode = "<redaction failed: code omitted>";
     }
-    emitError(opts.hooks, new Error(execResult.error || "compute execution failed"), {
+    emitError(hooks, new Error(execResult.error || "compute execution failed"), {
       stage: "compute_exec",
       code: maskedCode
     });
@@ -6535,7 +6632,7 @@ ${schemaLines.join("\n")}`;
   return redactValueRecursive(result, opts.config.redaction, {
     principals,
     secretKey: opts.config.secretKey,
-    hooks: opts.hooks
+    hooks
   });
 }
 var MAX_LIST_LIMIT, MAX_COMPUTE_TEXT_CHARS, DEFAULT_COMPUTE_TIMEOUT, MAX_COMPUTE_DOCUMENTS, TABULAR_MIME_PATTERNS, UUID_RE3, WORD_RE, STOPWORDS, SHEET_MARKER_RE2, CODE_FENCE_RE, COMPUTE_SYSTEM_PROMPT, visible;
@@ -6661,20 +6758,6 @@ function buildOpenAIClient(cfg2) {
     maxRetries: 0
   });
 }
-async function loadGeminiEmbedClient(apiKey) {
-  const specifier = "@google/genai";
-  let mod;
-  try {
-    mod = await import(specifier);
-  } catch {
-    throw new ExtraMissingError("gemini", specifier, "gemini embeddings");
-  }
-  const Ctor = mod.GoogleGenAI ?? mod.Client;
-  if (!Ctor) {
-    throw new ExtraMissingError("gemini", specifier, "gemini embeddings");
-  }
-  return new Ctor({ apiKey: apiKey ?? null, httpOptions: { timeout: TIMEOUT_MS3 } });
-}
 function buildEmbedder(cfg2, opts) {
   if (OPENAI_FAMILY2.has(cfg2.provider)) {
     return new Embedder(cfg2, { client: buildOpenAIClient(cfg2) });
@@ -6682,18 +6765,19 @@ function buildEmbedder(cfg2, opts) {
   if (cfg2.provider === "voyage" || cfg2.provider === "cohere") {
     return new Embedder(cfg2, { fetch: globalThis.fetch });
   }
-  if (cfg2.provider === "gemini") {
+  if (GOOGLE_FAMILY2.has(cfg2.provider)) {
     return new Embedder(cfg2);
   }
   throw new Error(`unknown embedding provider: ${JSON.stringify(cfg2.provider)}`);
 }
-var TIMEOUT_MS3, OPENAI_FAMILY2, Embedder;
+var TIMEOUT_MS3, OPENAI_FAMILY2, GOOGLE_FAMILY2, Embedder;
 var init_embeddings = __esm({
   "src/providers/embeddings.ts"() {
-    init_errors();
     init_text();
+    init_google();
     TIMEOUT_MS3 = 3e4;
     OPENAI_FAMILY2 = /* @__PURE__ */ new Set(["openai", "azure_openai", "custom"]);
+    GOOGLE_FAMILY2 = /* @__PURE__ */ new Set(["gemini", "vertex_ai"]);
     Embedder = class {
       cfg;
       provider;
@@ -6741,7 +6825,7 @@ var init_embeddings = __esm({
       /** Provider dispatch. Overridable per-instance (tests stub this). */
       async rawEmbed(texts, kind = "document") {
         if (OPENAI_FAMILY2.has(this.provider)) return this.embedOpenAI(texts);
-        if (this.provider === "gemini") return this.embedGemini(texts);
+        if (GOOGLE_FAMILY2.has(this.provider)) return this.embedGemini(texts);
         if (this.provider === "voyage") return this.embedVoyage(texts, kind);
         if (this.provider === "cohere") return this.embedCohere(texts, kind);
         throw new Error(`unknown embedding provider: ${JSON.stringify(this.provider)}`);
@@ -6757,7 +6841,7 @@ var init_embeddings = __esm({
       }
       async embedGemini(texts) {
         if (!this.genaiClient) {
-          this.genaiClient = await loadGeminiEmbedClient(this.cfg.apiKey);
+          this.genaiClient = await buildGenaiClient(this.cfg, TIMEOUT_MS3, "gemini embeddings");
         }
         const resp = await this.genaiClient.models.embedContent({
           model: this.model,