@jangmanj/pi-webtasks 0.1.0

package/src/env.ts

@@ -0,0 +1,68 @@
+import { readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+/**
+ * Loads key files adjacent to the extension (`<extension-root>/.env` and `<extension-root>/parallel.env`)
+ * into process.env before any tool runs. Real environment variables always win; nothing here overrides a
+ * set variable. The research key (_PARALLEL_API_KEY or PARALLEL_API_KEY) is held aside instead, so the
+ * environment and Pi's stored login both win over it; see `packageAdjacentKey`.
+ * No Pi imports: callable from a plain script as well.
+ */
+
+/** Parse dotenv-style text. Supports blank lines, `#` comments, `export KEY=`, and quoted values. */
+export function parseEnvText(text: string): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const rawLine of text.split("\n")) {
+    const line = rawLine.trim();
+    if (!line || line.startsWith("#")) continue;
+    const m = /^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
+    if (!m) continue;
+    let value = m[2].trim();
+    const quote = value[0];
+    if ((quote === '"' || quote === "'") && value.length > 1 && value.endsWith(quote)) {
+      value = value.slice(1, -1);
+    } else {
+      value = value.replace(/\s+#.*$/, "").trimEnd();
+    }
+    out[m[1]] = value;
+  }
+  return out;
+}
+
+/** Apply entries without overriding variables that are already set. An explicit empty value counts as set. */
+export function applyEnv(entries: Record<string, string>, env: Record<string, string | undefined> = process.env): void {
+  for (const [key, value] of Object.entries(entries)) {
+    if (env[key] === undefined) env[key] = value;
+  }
+}
+
+/** This module lives in `src/`; key files live in the extension root. */
+const EXTENSION_ROOT = dirname(dirname(fileURLToPath(import.meta.url)));
+
+/**
+ * Research keys found in the package-adjacent files. They are kept out of process.env, so a variable set in
+ * the environment always wins over them. The first file that defines a key wins.
+ */
+const packageKeys: Record<string, string> = {};
+const KEY_NAMES = new Set(["_PARALLEL_API_KEY", "PARALLEL_API_KEY"]);
+
+for (const name of [".env", "parallel.env"]) {
+  let entries: Record<string, string>;
+  try {
+    entries = parseEnvText(readFileSync(join(EXTENSION_ROOT, name), "utf8"));
+  } catch {
+    continue; /* no readable key file at this path */
+  }
+  const settings: Record<string, string> = {};
+  for (const [key, value] of Object.entries(entries)) {
+    if (KEY_NAMES.has(key)) packageKeys[key] ??= value;
+    else settings[key] = value;
+  }
+  applyEnv(settings);
+}
+
+/** The research key from the package-adjacent key file, if any. */
+export function packageAdjacentKey(): string | undefined {
+  return [...KEY_NAMES].map(name => packageKeys[name]?.trim()).find(Boolean) || undefined;
+}

package/src/index.ts

@@ -0,0 +1,175 @@
+/**
+ * pi-webtasks: session-bound web research for Pi.
+ *
+ * The extension owns transport, the local evidence store, indexing, recovery,
+ * and alert delivery. The model owns the research work: what to search, what to
+ * read, and what to ask. Nothing here exposes SQL, provider ids, or file paths.
+ *
+ * Collection runs inside Pi. There is no webhook, no callback receiver, and no
+ * artifact file: received evidence is committed to a local SQLite store owned by
+ * the Pi session. If Pi closes, collection stops; the next session start
+ * reconciles provider state where retention permits.
+ */
+
+import { getAgentDir, type ExtensionAPI, type ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { join } from "node:path";
+import { formatError, researchClient, type StoredKey } from "./ops.ts";
+import { PARALLEL_PROVIDER_ID, parallelProvider } from "./provider.ts";
+import { ResearchStore, type AlertRecord } from "./store.ts";
+import { createResearchService, type ResearchService } from "./research-service.ts";
+import { createRecallService, type RecallService } from "./recall.ts";
+import { createTaskService, type TaskService } from "./task-service.ts";
+import { createAnswerService, type AnswerService } from "./answer-service.ts";
+import { registerResearchTools, type ResearchServices } from "./research-tools.ts";
+import { createAlertDelivery, type AlertDelivery } from "./alert-delivery.ts";
+import { WebtasksStatus } from "./status.ts";
+
+/** Optional explicit store directory. */
+const WEBTASKS_DIR_ENV = "WEBTASKS_DIR";
+
+/** Explicitly configured store directory. */
+function configuredStoreDir(): string | undefined {
+  const dir = process.env[WEBTASKS_DIR_ENV];
+  return dir && dir.trim() ? dir : undefined;
+}
+
+/** Research evidence lives in WEBTASKS_DIR when set, else `<agent-dir>/webtasks`. */
+function storeDirectory(): string {
+  return configuredStoreDir() ?? join(getAgentDir(), "webtasks");
+}
+
+/** Everything owned by one started Pi session. A reload or session switch builds a new one. */
+type Runtime = {
+  sessionId: string;
+  store: ResearchStore;
+  research: ResearchService;
+  recall: RecallService;
+  task: TaskService;
+  answer: AnswerService;
+  /** Hands research alerts to Pi as follow-ups, at least once. */
+  alerts: AlertDelivery;
+  /** Footer status (TUI only): web, answer, and research counts with estimated spend. */
+  status?: WebtasksStatus;
+  released: boolean;
+};
+
+export default function piWebtasks(pi: ExtensionAPI) {
+  let rt: Runtime | undefined;
+
+  // Registered like any provider. /login lists it under "Sign in with an API key".
+  pi.registerProvider(parallelProvider());
+
+  async function stopRuntime(): Promise<void> {
+    const outgoing = rt;
+    if (!outgoing) return;
+    // Mark the runtime unusable first: nothing may write to a store that is closing.
+    outgoing.released = true;
+    if (rt === outgoing) rt = undefined;
+    outgoing.status?.close();
+    await outgoing.alerts.stop();
+    await outgoing.task.stop().catch(() => undefined);
+    await outgoing.store.close().catch(() => undefined);
+  }
+
+  function currentServices(): ResearchServices {
+    if (!rt || rt.released) throw new Error("Research tools are unavailable: the session is not started");
+    return { research: rt.research, recall: rt.recall, task: rt.task, answer: rt.answer, status: rt.status };
+  }
+
+  /**
+   * The Pi side of alert delivery: an idle session wakes, a busy one receives
+   * the alert after its current work. Throwing leaves the alert to be offered again.
+   */
+  function alertSink(ctx: ExtensionContext) {
+    return (alert: AlertRecord) => {
+      const text = [alert.headline, alert.body].filter((line): line is string => Boolean(line)).join("\n");
+      if (ctx.hasUI) ctx.ui.notify(text, alert.kind === "completion" ? "info" : "warning");
+      pi.sendMessage(
+        { customType: "webtasks-research", content: text, display: true, details: { alert_id: alert.alert_id, reference: alert.reference, kind: alert.kind } },
+        { deliverAs: "followUp", triggerTurn: true },
+      );
+    };
+  }
+
+  pi.on("session_start", async (_event, ctx) => {
+    await stopRuntime();
+    const sessionId = ctx.sessionManager.getSessionId();
+    // The stored login for this provider, read on every request. Nothing about the key is cached here.
+    const modelRegistry = ctx.modelRegistry;
+    const storedKey: StoredKey = async () => (await modelRegistry?.getApiKeyForProvider(PARALLEL_PROVIDER_ID)) ?? undefined;
+
+    let store: ResearchStore;
+    try {
+      store = await new ResearchStore(storeDirectory()).init();
+    } catch (err) {
+      if (ctx.hasUI) ctx.ui.notify(`pi-webtasks: research store unavailable: ${formatError(err)}`, "warning");
+      return;
+    }
+    try {
+      await store.registerSession(sessionId);
+      // A request interrupted by a crash or a close has no recovery path here: mark it
+      // uncertain instead of leaving it silently unfinished. Best effort.
+      await store.reconcileInterrupted(sessionId).catch(() => undefined);
+    } catch (err) {
+      if (ctx.hasUI) ctx.ui.notify(`pi-webtasks: research store unavailable: ${formatError(err)}`, "warning");
+      await store.close().catch(() => undefined);
+      return;
+    }
+
+    const model = ctx.model as { provider?: string; id?: string } | undefined;
+    const client = async () => await researchClient(storedKey);
+    const alerts = createAlertDelivery({
+      store,
+      sessionId,
+      deliver: alertSink(ctx),
+      onError: err => { if (ctx.hasUI) ctx.ui.notify(`pi-webtasks: alert delivery failed: ${formatError(err)}`, "warning"); },
+    });
+    const runtime: Runtime = {
+      sessionId,
+      store,
+      research: createResearchService({
+        store,
+        sessionId,
+        client,
+        clientModel: model?.provider && model?.id ? `${model.provider}/${model.id}` : undefined,
+      }),
+      recall: createRecallService({ store, sessionId, alerts }),
+      task: createTaskService({
+        store,
+        sessionId,
+        client,
+        onAlerts: () => {
+          // Progress and completion change what the footer shows: runs in progress, sources, spend.
+          void runtime.status?.refresh();
+          void alerts.wake();
+        },
+        log: message => { if (ctx.hasUI) ctx.ui.notify(`pi-webtasks: ${message}`, "warning"); },
+        // Testing aid: shorten the progress sweep without changing the default.
+        ...(Number(process.env.WEBTASKS_SWEEP_MS) >= 500 ? { sweepMs: Number(process.env.WEBTASKS_SWEEP_MS) } : {}),
+      }),
+      answer: createAnswerService({ store, sessionId, client }),
+      alerts,
+      released: false,
+    };
+    rt = runtime;
+    if (ctx.hasUI && ctx.mode === "tui") {
+      runtime.status = new WebtasksStatus(text => ctx.ui.setStatus("webtasks", text), () => store.runSummaries(sessionId));
+      void runtime.status.refresh();
+    }
+    runtime.task.start();
+    // A new runtime owns delivery: what the previous one left unacknowledged is offered again.
+    void alerts.start();
+    // Reconcile in the background: a settled run must not delay session start.
+    void runtime.task.reconcile().then(() => runtime.status?.refresh()).catch(err => {
+      if (ctx.hasUI) ctx.ui.notify(`pi-webtasks: research reconciliation failed: ${formatError(err)}`, "warning");
+    });
+  });
+
+  pi.on("session_shutdown", async () => {
+    await stopRuntime();
+  });
+
+  // Discovery, reading, recall, background research, and answers resolve the store
+  // through the current runtime, not a captured one.
+  registerResearchTools(pi, currentServices);
+}

package/src/ops.ts

@@ -0,0 +1,64 @@
+/**
+ * Parallel client plumbing shared by every research call.
+ *
+ * The key precedence rule lives here so the extension, tests, and any future
+ * caller resolve credentials the same way. Nothing here retries: the client is
+ * built with `maxRetries: 0`, and call sites pass it explicitly as well.
+ */
+
+import { packageAdjacentKey } from "./env.ts";
+import { Parallel, APIError } from "parallel-web";
+
+/** Where a key stored by Pi's /login comes from. Absent when the caller runs without Pi. */
+export type StoredKey = () => Promise<string | undefined>;
+
+/** Shown when Pi runs the extension. The sentence names the one step a user can take inside Pi. */
+export const MISSING_KEY_IN_PI = "Run /login, choose Parallel, and paste an API key — or set _PARALLEL_API_KEY.";
+/** Shown when there is no Pi login, so the only sources are the environment and the package key file. */
+export const MISSING_KEY_STANDALONE = "Parallel API key missing. Set _PARALLEL_API_KEY in the environment, or put it in .env or parallel.env beside the package manifest.";
+
+export type KeySource = "stored credential" | "environment" | "key file";
+
+/**
+ * The one precedence rule for the research key: the stored login, then _PARALLEL_API_KEY, then
+ * PARALLEL_API_KEY from the environment, then the package-adjacent key file. Pure, so tests can drive it.
+ */
+export function resolveResearchKey(input: {
+  stored?: string;
+  env?: Record<string, string | undefined>;
+  packageKey?: string;
+}): { key: string; source: KeySource } | undefined {
+  const stored = input.stored?.trim();
+  if (stored) return { key: stored, source: "stored credential" };
+  const env = input.env ?? process.env;
+  for (const name of ["_PARALLEL_API_KEY", "PARALLEL_API_KEY"]) {
+    const value = env[name]?.trim();
+    if (value) return { key: value, source: "environment" };
+  }
+  const file = input.packageKey?.trim();
+  return file ? { key: file, source: "key file" } : undefined;
+}
+
+/** Resolved on every call, never cached, so a login or logout takes effect on the next request. */
+export async function researchKey(stored?: StoredKey): Promise<string> {
+  const resolved = resolveResearchKey({ stored: stored ? await stored() : undefined, packageKey: packageAdjacentKey() });
+  if (!resolved) throw new Error(stored ? MISSING_KEY_IN_PI : MISSING_KEY_STANDALONE);
+  return resolved.key;
+}
+
+export async function researchClient(stored?: StoredKey): Promise<Parallel> {
+  return new Parallel({ apiKey: await researchKey(stored), timeout: 120_000, maxRetries: 0 });
+}
+
+/** A readable message for a provider or local failure, without leaking credentials. */
+export function formatError(err: unknown): string {
+  if (err instanceof APIError) {
+    try {
+      return JSON.stringify(err.error ?? err.message, null, 2);
+    } catch {
+      return err.message;
+    }
+  }
+  if (err instanceof Error) return err.message;
+  return String(err);
+}

package/src/paid-call.ts

@@ -0,0 +1,202 @@
+/**
+ * The recorded paid call: one request to Parallel that costs money, recorded
+ * before it is sent and settled exactly once.
+ *
+ * Every paid tool goes through here (web_search, web_fetch, web_answer, and the
+ * web_research submission), so these rules hold in one place:
+ *
+ *  - Intent before money: the run is recorded, keyed by the Pi tool invocation,
+ *    before the request. A replayed invocation returns the stored run and is
+ *    never sent again.
+ *  - A missing or unusable credential is a local failure: the provider was never
+ *    asked, so the run is `failed` and the original, actionable message is kept.
+ *  - The request is sent once and never retried (`maxRetries: 0`).
+ *  - A 4xx response is a definite rejection: `failed`. Anything else (5xx, 408,
+ *    a lost connection, a timeout, an abort, an unexpected error once the
+ *    request was under way) may have reached the provider, so the run is
+ *    `uncertain` and is never silently resubmitted.
+ *  - A usable response is committed with its evidence and citations in one
+ *    store transaction. A response that arrived but cannot be used settles the
+ *    run as the caller says (`UnusableResponse`).
+ *  - A call without `settle` hands the accepted run off (Task research): the run
+ *    keeps the state that `send` recorded, and collection settles it later.
+ *
+ * The caller owns what varies: input validation, the request, how a response
+ * becomes evidence, and the wording of its receipt.
+ */
+
+import type { CitationInput, EvidenceInput, ResearchStore, RunKind, RunRecord, SearchHit, StoredEvidence } from "./store.ts";
+import { errorText } from "./receipt.ts";
+
+/** Upper bound for one synchronous paid request. */
+export const REQUEST_TIMEOUT_MS = 180_000;
+/** Stored evidence listed for a replayed invocation. */
+const REPLAY_HITS = 20;
+
+/** Options every paid request is sent with. */
+export type RequestOptions = { signal?: AbortSignal; maxRetries: 0; timeout: number };
+
+/** What a usable response stores, committed in one transaction with the completed state. */
+export type Settlement = {
+  summary: string;
+  evidence?: EvidenceInput[];
+  citations?: CitationInput[];
+  /** Provider operation id (search id, extract id, response id). */
+  providerRunId?: string;
+  /** Provider research context (Search/Extract session id). */
+  providerSessionId?: string;
+  /** Provider interaction id, for a later continuation. */
+  providerInteractionId?: string;
+  /** Untouched provider response, stored verbatim. */
+  rawResult: unknown;
+};
+
+export type PaidCall<C, R, I extends object = Record<string, unknown>> = {
+  kind: RunKind;
+  /** Names the call in errors: "Search", "Fetch", "Answer", "Research submission". */
+  label: string;
+  /** The effective request, recorded before it is sent. A replay returns it as stored. */
+  intent: I;
+  processor?: string;
+  /** The Pi tool invocation: its id keys the record, its signal may abort the request. */
+  invocation: { invocationId: string; signal?: AbortSignal };
+  /** The one paid request. It is never retried; `reference` names the local record. */
+  send(client: C, request: { reference: string; options: RequestOptions }): Promise<R>;
+  /** What a usable response stores. Omit it to hand the accepted run off to collection. */
+  settle?(response: R, reference: string): Settlement;
+  /** The advice after an outcome-unknown failure. Default: try again only if the result is still needed. */
+  uncertainAdvice?(reference: string): string;
+};
+
+export type PaidOutcome<R, I extends object = Record<string, unknown>> =
+  | {
+    replayed: true;
+    reference: string;
+    /** The stored run; absent only if it vanished between the lookup and the read. */
+    run?: RunRecord;
+    /**
+     * The intent the first invocation recorded, not the replay's arguments. A row
+     * written by an older release may lack fields: read each with a fallback.
+     */
+    intent?: Partial<I>;
+    /** Evidence already stored for it, newest first. */
+    hits: SearchHit[];
+  }
+  | {
+    replayed: false;
+    reference: string;
+    response: R;
+    /** What `settle` committed; absent for a hand-off. */
+    settlement?: Settlement;
+    /** The stored evidence, each with its new handle and its url and pointer, in the order `settle` gave; empty for a hand-off. */
+    evidence: StoredEvidence[];
+  };
+
+/**
+ * Thrown by `settle` when a response arrived but cannot be used. The run is
+ * settled as `state` with `reason` recorded, and the agent sees `message`.
+ */
+export class UnusableResponse extends Error {
+  readonly state: "failed" | "uncertain";
+  readonly reason: string;
+  constructor(state: "failed" | "uncertain", reason: string, message: string) {
+    super(message);
+    this.name = "UnusableResponse";
+    this.state = state;
+    this.reason = reason;
+  }
+}
+
+/** A 4xx response is a definite rejection; anything else may have reached the provider. */
+export function outcomeUnknown(err: unknown): boolean {
+  const status = (err as { status?: unknown } | null | undefined)?.status;
+  return typeof status === "number" ? status >= 500 || status === 408 : true;
+}
+
+/** Longest failure message recorded or shown; provider error bodies can be large. */
+const MESSAGE_LIMIT = 500;
+
+/** The readable message of a failure, on one bounded line. */
+export function failureMessage(err: unknown): string {
+  return errorText(err, MESSAGE_LIMIT) ?? "unknown error";
+}
+
+function defaultAdvice(reference: string): string {
+  return `Its outcome is unknown (record ${reference}); try again only if you still need the result.`;
+}
+
+/** Paid calls for one Pi session: one store, one session id, one provider client factory. */
+export function createPaidCalls<C>(options: { store: ResearchStore; sessionId: string; client: () => Promise<C> }) {
+  const { store, sessionId } = options;
+
+  /** Best effort: a store that cannot record the outcome must not hide the original error. */
+  async function settleAs(reference: string, state: "failed" | "uncertain", error: string): Promise<void> {
+    await store.setRunState({ sessionId, reference, state, error }).catch(() => undefined);
+  }
+
+  async function run<R, I extends object = Record<string, unknown>>(call: PaidCall<C, R, I>): Promise<PaidOutcome<R, I>> {
+    const begun = await store.beginRun({
+      sessionId,
+      intentKey: call.invocation.invocationId,
+      intent: call.intent,
+      kind: call.kind,
+      processor: call.processor,
+      state: "submitting",
+    });
+    const reference = begun.reference;
+    if (!begun.created) {
+      const stored = await store.run(sessionId, reference);
+      const page = await store.search(sessionId, { reference, limit: REPLAY_HITS });
+      const intent = stored?.intent && typeof stored.intent === "object" ? stored.intent as Partial<I> : undefined;
+      return { replayed: true, reference, run: stored, intent, hits: page.hits };
+    }
+
+    let client: C;
+    try {
+      client = await options.client();
+    } catch (err) {
+      await settleAs(reference, "failed", failureMessage(err));
+      throw err;
+    }
+
+    let response: R;
+    try {
+      response = await call.send(client, {
+        reference,
+        options: { signal: call.invocation.signal, maxRetries: 0, timeout: REQUEST_TIMEOUT_MS },
+      });
+    } catch (err) {
+      const uncertain = outcomeUnknown(err);
+      const message = failureMessage(err);
+      await settleAs(reference, uncertain ? "uncertain" : "failed", message);
+      throw new Error(
+        `${call.label} did not complete: ${message}. ${uncertain ? (call.uncertainAdvice ?? defaultAdvice)(reference) : `Record ${reference}.`}`,
+      );
+    }
+
+    if (!call.settle) return { replayed: false, reference, response, evidence: [] };
+    try {
+      const settlement = call.settle(response, reference);
+      const completion = await store.completeRun({ sessionId, reference, state: "completed", ...settlement });
+      return { replayed: false, reference, response, settlement, evidence: completion.evidence };
+    } catch (err) {
+      // The provider answered, so nothing here may claim the request failed outright,
+      // unless the caller judged the response itself.
+      if (err instanceof UnusableResponse) {
+        await settleAs(reference, err.state, err.reason);
+        throw err;
+      }
+      const message = failureMessage(err);
+      await settleAs(reference, "uncertain", `the response could not be stored: ${message}`);
+      // Say that it ran: an agent that reads a bare storage error would pay for it again.
+      throw new Error(
+        `${call.label} ran, but its result could not be stored: ${message}. Record ${reference} is uncertain; repeat the request only if you still need the result.`,
+        { cause: err },
+      );
+    }
+  }
+
+  return { run };
+}
+
+export type PaidCalls<C> = ReturnType<typeof createPaidCalls<C>>;