@h1v35/hivex 0.1.0

package/src/markdown.ts

@@ -0,0 +1,98 @@
+import { createHash } from 'node:crypto';
+import { basename } from 'node:path';
+import { fromMarkdown } from 'mdast-util-from-markdown';
+import { toString } from 'mdast-util-to-string';
+import { gfmFromMarkdown } from 'mdast-util-gfm';
+import { gfm } from 'micromark-extension-gfm';
+import { frontmatterFromMarkdown } from 'mdast-util-frontmatter';
+import { frontmatter } from 'micromark-extension-frontmatter';
+import { normalizeIdentifier } from 'micromark-util-normalize-identifier';
+import { parseDocument } from 'yaml';
+
+export const hash = (text: string) => createHash('sha256').update(text).digest('hex');
+export const isMarkdownPath = (path: string) => /\.(?:md|markdown|mdown)$/i.test(path);
+
+export function parseMarkdown(content: string) {
+  return fromMarkdown(content, {
+    extensions: [gfm(), frontmatter(['yaml'])],
+    mdastExtensions: [gfmFromMarkdown(), frontmatterFromMarkdown(['yaml'])],
+  });
+}
+
+type MarkdownNode = {
+  type: string;
+  children?: MarkdownNode[];
+  url?: string;
+  identifier?: string;
+  position?: { start: { line: number; offset?: number }; end: { line: number; offset?: number } };
+};
+
+export function* descendants(tree: MarkdownNode): Generator<MarkdownNode> {
+  const pending = [tree];
+  while (pending.length) {
+    const node = pending.pop();
+    if (!node) break;
+    yield node;
+    for (let index = (node.children?.length ?? 0) - 1; index >= 0; index--) {
+      const child = node.children?.[index];
+      if (child) pending.push(child);
+    }
+  }
+}
+
+function metadata(tree: ReturnType<typeof parseMarkdown>) {
+  const node = tree.children.find((entry) => entry.type === 'yaml');
+  if (node?.type !== 'yaml') return { title: null, status: null };
+  try {
+    const parsed = parseDocument(node.value, { uniqueKeys: true });
+    if (parsed.errors.length) return { title: null, status: null };
+    const value: unknown = parsed.toJS({ maxAliasCount: 0 });
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+      return { title: null, status: null };
+    return {
+      title:
+        'title' in value && typeof value.title === 'string' && value.title.trim()
+          ? value.title
+          : null,
+      status: 'status' in value && typeof value.status === 'string' ? value.status : null,
+    };
+  } catch {
+    return { title: null, status: null };
+  }
+}
+
+export function describeMarkdown(path: string, content: string) {
+  const tree = parseMarkdown(content);
+  const front = metadata(tree);
+  const heading = tree.children.find((node) => node.type === 'heading');
+  const definitions = new Map<string, string>();
+  for (const node of descendants(tree)) {
+    if (node.type === 'definition' && node.identifier && node.url) {
+      const id = normalizeIdentifier(node.identifier);
+      if (!definitions.has(id)) definitions.set(id, node.url);
+    }
+  }
+  const links = [...descendants(tree)].flatMap((node) => {
+    if (node.type === 'link' && node.url) return [node.url];
+    if (node.type !== 'linkReference' || !node.identifier) return [];
+    const url = definitions.get(normalizeIdentifier(node.identifier));
+    return url ? [url] : [];
+  });
+  return {
+    title: front.title ?? (heading ? toString(heading) : basename(path)),
+    status: front.status,
+    links,
+  };
+}
+
+export function rawMarkdownLines(text: string): string[] {
+  const lines = (text.match(/[^\r\n]*(?:\r\n|\r|\n|$)/g) ?? []).filter((line) => line.length > 0);
+  return lines.length ? lines : [''];
+}
+
+export const lineContent = (line: string) => line.replace(/(?:\r\n|\r|\n)$/, '');
+
+export function sourceRange(text: string, from: number, to: number) {
+  const selected = rawMarkdownLines(text).slice(from - 1, to);
+  return selected.slice(0, -1).join('') + lineContent(selected.at(-1) ?? '');
+}

package/src/model/connection.ts

@@ -0,0 +1,207 @@
+import { createInterface, type Interface } from 'node:readline';
+import type { Readable, Writable } from 'node:stream';
+import { z } from 'zod';
+
+const IdSchema = z.union([z.string(), z.number().int()]);
+const RemoteErrorSchema = z.looseObject({
+  code: z.number().int(),
+  message: z.string(),
+  data: z.unknown().optional(),
+});
+const FrameSchema = z.looseObject({
+  id: IdSchema.optional(),
+  method: z.string().optional(),
+  params: z.unknown().optional(),
+  result: z.unknown().optional(),
+  error: RemoteErrorSchema.optional(),
+});
+type Frame = z.infer<typeof FrameSchema>;
+
+interface ConnectionOptions {
+  readonly input: Writable;
+  readonly output: Readable;
+  readonly onNotification: (method: string, params: unknown) => void;
+  readonly onInteractiveRequest?: (method: string) => void;
+}
+interface RequestOptions {
+  readonly timeoutMilliseconds?: number;
+  // Cancels the local wait. Remote turns require an explicit turn/interrupt request.
+  readonly signal?: AbortSignal;
+}
+interface PendingRequest {
+  readonly resolve: (value: unknown) => void;
+  readonly reject: (error: unknown) => void;
+  readonly cleanup: () => void;
+}
+
+export class AppServerRpcError extends Error {
+  readonly code: number;
+  readonly data: unknown;
+
+  constructor(error: z.infer<typeof RemoteErrorSchema>) {
+    super(error.message);
+    this.name = 'AppServerRpcError';
+    this.code = error.code;
+    this.data = error.data;
+  }
+}
+
+export class AppServerConnection {
+  private readonly options: ConnectionOptions;
+  private readonly reader: Interface;
+  private readonly pending = new Map<string | number, PendingRequest>();
+  private readonly ending = Promise.withResolvers<never>();
+  private failure: Error | undefined;
+  private nextId = 1;
+  private trailingBytes = 0;
+  private streamBytes = 0;
+  readonly closed = this.ending.promise;
+
+  constructor(options: ConnectionOptions) {
+    this.options = options;
+    options.output.on('data', this.measureChunk);
+    this.reader = createInterface({ input: options.output, crlfDelay: Infinity });
+    this.reader.on('line', this.receiveLine);
+    this.reader.on('close', this.receiveClose);
+    options.input.on('error', this.receiveStreamError);
+    options.output.on('error', this.receiveStreamError);
+    void this.closed.catch(() => undefined);
+  }
+
+  request(method: string, params: unknown, options: RequestOptions = {}): Promise<unknown> {
+    if (this.failure !== undefined) return Promise.reject(this.failure);
+    if (options.signal?.aborted) return Promise.reject(new Error('request cancelled'));
+    const timeout = options.timeoutMilliseconds ?? 30_000;
+    if (!Number.isSafeInteger(timeout) || timeout <= 0) {
+      return Promise.reject(new Error('request timeout must be a positive integer'));
+    }
+    const id = this.nextId++;
+    const result = Promise.withResolvers<unknown>();
+    const abort = () => this.rejectRequest(id, new Error('request cancelled'));
+    const timer = setTimeout(
+      () => this.rejectRequest(id, new Error(`request timed out: ${method}`)),
+      timeout,
+    );
+    this.pending.set(id, {
+      resolve: result.resolve,
+      reject: result.reject,
+      cleanup: () => {
+        clearTimeout(timer);
+        options.signal?.removeEventListener('abort', abort);
+      },
+    });
+    options.signal?.addEventListener('abort', abort, { once: true });
+    this.send({ id, method, params });
+    return result.promise;
+  }
+
+  notify(method: string, params?: unknown): void {
+    if (this.failure !== undefined) throw this.failure;
+    this.send({ method, params });
+  }
+
+  dispose(): void {
+    this.fail(new Error('connection closed'));
+    this.reader.off('line', this.receiveLine);
+    this.reader.off('close', this.receiveClose);
+    this.reader.close();
+    this.options.input.off('error', this.receiveStreamError);
+    this.options.output.off('error', this.receiveStreamError);
+    this.options.output.off('data', this.measureChunk);
+  }
+
+  private readonly receiveClose = (): void => this.fail(new Error('connection closed'));
+  private readonly receiveStreamError = (error: Error): void => this.fail(error);
+
+  private readonly measureChunk = (chunk: Buffer): void => {
+    if (this.failure) return;
+    this.streamBytes += chunk.byteLength;
+    if (this.streamBytes > 33_554_432) {
+      this.fail(new Error('app-server stream exceeded 32 MiB'));
+      return;
+    }
+    let offset = 0;
+    for (let newline = chunk.indexOf(10); newline !== -1; newline = chunk.indexOf(10, offset)) {
+      this.trailingBytes += newline - offset;
+      if (this.trailingBytes > 4_194_304) {
+        this.fail(new Error('app-server frame exceeded 4 MiB'));
+        return;
+      }
+      this.trailingBytes = 0;
+      offset = newline + 1;
+    }
+    this.trailingBytes += chunk.byteLength - offset;
+    if (this.trailingBytes > 4_194_304) this.fail(new Error('app-server frame exceeded 4 MiB'));
+  };
+
+  private readonly receiveLine = (line: string): void => {
+    if (this.failure !== undefined) return;
+    try {
+      const frame = FrameSchema.parse(JSON.parse(line) as unknown);
+      this.receiveFrame(frame);
+    } catch (error: unknown) {
+      this.fail(new Error('invalid app-server frame', { cause: error }));
+    }
+  };
+
+  private receiveFrame(frame: Frame): void {
+    if (frame.method !== undefined) {
+      if (Object.hasOwn(frame, 'result') || frame.error !== undefined) {
+        throw new Error('a method frame cannot also be a response');
+      }
+      if (frame.id === undefined) this.options.onNotification(frame.method, frame.params);
+      else this.declineInteractiveRequest(frame.id, frame.method);
+      return;
+    }
+    if (frame.id === undefined || Object.hasOwn(frame, 'result') === (frame.error !== undefined)) {
+      throw new Error('a response requires an id and exactly one outcome');
+    }
+    const pending = this.pending.get(frame.id);
+    if (pending === undefined) return;
+    this.pending.delete(frame.id);
+    pending.cleanup();
+    if (frame.error !== undefined) pending.reject(new AppServerRpcError(frame.error));
+    else pending.resolve(frame.result);
+  }
+
+  private declineInteractiveRequest(id: string | number, method: string): void {
+    this.options.onInteractiveRequest?.(method);
+    if (
+      method === 'item/commandExecution/requestApproval' ||
+      method === 'item/fileChange/requestApproval'
+    ) {
+      this.send({ id, result: { decision: 'decline' } });
+      return;
+    }
+    this.send({
+      id,
+      error: { code: -32601, message: 'interactive requests are outside this pilot' },
+    });
+  }
+
+  private send(frame: unknown): void {
+    try {
+      this.options.input.write(`${JSON.stringify(frame)}\n`, (error) => {
+        if (error !== null && error !== undefined) this.fail(error);
+      });
+    } catch (error: unknown) {
+      this.fail(new Error('app-server write failed', { cause: error }));
+    }
+  }
+
+  private rejectRequest(id: string | number, error: Error): void {
+    const pending = this.pending.get(id);
+    if (pending === undefined) return;
+    this.pending.delete(id);
+    pending.cleanup();
+    pending.reject(error);
+  }
+
+  private fail(error: Error): void {
+    if (this.failure !== undefined) return;
+    this.failure = error;
+    this.reader.close();
+    for (const id of this.pending.keys()) this.rejectRequest(id, error);
+    this.ending.reject(error);
+  }
+}

package/src/model/failure.ts

@@ -0,0 +1,33 @@
+import { z } from 'zod';
+import { AppServerRpcError } from './connection.ts';
+import type { ProfileEvidence } from './profile.ts';
+
+export class ServerAdmissionFailure extends Error {
+  readonly cleanup: 'confirmed' | 'failed';
+  readonly processId: number;
+  readonly admission: (ProfileEvidence & { launchPolicyHash: string }) | undefined;
+  constructor(options: {
+    cause: unknown;
+    cleanup: 'confirmed' | 'failed';
+    processId: number;
+    admission?: ProfileEvidence & { launchPolicyHash: string };
+  }) {
+    super('Native server admission failed', { cause: options.cause });
+    this.cleanup = options.cleanup;
+    this.processId = options.processId;
+    this.admission = options.admission;
+  }
+}
+
+export function failureDiagnostic(error: unknown): Record<string, unknown> {
+  if (error instanceof ServerAdmissionFailure) return failureDiagnostic(error.cause);
+  if (error instanceof z.ZodError)
+    return {
+      kind: 'invalid-native-response',
+      fields: error.issues.map((issue) => issue.path.join('.')),
+    };
+  if (error instanceof AppServerRpcError) return { kind: 'rpc-rejection', code: error.code };
+  if (error instanceof Error && error.constructor === Error)
+    return { kind: 'native-admission', message: error.message };
+  return { kind: 'native-failure' };
+}

package/src/model/invoke.ts

@@ -0,0 +1,265 @@
+import { mkdtempSync, readdirSync, realpathSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { z } from 'zod';
+import { startServer } from './server.ts';
+import { knowledgeTurn, type ProfileEvidence } from './profile.ts';
+import { captureTranscript, type Usage } from './transcript.ts';
+import { startKnowledgeThread } from './thread.ts';
+import { failureDiagnostic, ServerAdmissionFailure } from './failure.ts';
+
+export type NativeProcessStarted = (nativeProcessId: number) => void;
+
+export type InvocationOptions = {
+  binary: string;
+  prompt: string;
+  schema: Record<string, unknown>;
+  deadlineMilliseconds: number;
+  onNativeProcessStarted?: NativeProcessStarted;
+};
+export type InvocationReport = {
+  outcome: string;
+  code?: string;
+  deadlineMilliseconds: number;
+  interruption?: string;
+  usage: Usage | null;
+  turnAccepted?: 'confirmed' | 'unknown';
+  threadId?: string;
+  turnId?: string;
+  cleanup?: 'confirmed' | 'failed' | 'not-observed';
+  startedAt?: string;
+  durationMilliseconds?: number;
+  admission?: ProfileEvidence & { launchPolicyHash: string };
+  diagnostic?: Record<string, unknown>;
+  nativeProcessId?: number;
+};
+
+class ModelTimeout extends Error {}
+class ModelCancelled extends Error {}
+
+async function completedWithin(options: {
+  captured: ReturnType<typeof captureTranscript>;
+  server: Awaited<ReturnType<typeof startServer>>;
+  milliseconds: number;
+  signal?: AbortSignal;
+}) {
+  const expired = Promise.withResolvers<never>();
+  const cancelled = () => expired.reject(new ModelCancelled());
+  options.signal?.addEventListener('abort', cancelled, { once: true });
+  if (options.signal?.aborted) cancelled();
+  const timer = setTimeout(() => expired.reject(new ModelTimeout()), options.milliseconds);
+  try {
+    return await Promise.race([
+      options.captured.done.promise,
+      options.server.rpc.closed,
+      expired.promise,
+    ]);
+  } finally {
+    clearTimeout(timer);
+    options.signal?.removeEventListener('abort', cancelled);
+  }
+}
+
+async function interrupt(options: {
+  captured: ReturnType<typeof captureTranscript>;
+  server: Awaited<ReturnType<typeof startServer>>;
+  threadId: string;
+  turnId: string;
+}) {
+  try {
+    await options.server.rpc.request(
+      'turn/interrupt',
+      { threadId: options.threadId, turnId: options.turnId },
+      { timeoutMilliseconds: 5000 },
+    );
+    const end = await completedWithin({
+      captured: options.captured,
+      server: options.server,
+      milliseconds: 5000,
+    });
+    return (
+      end.threadId === options.threadId &&
+      end.turn.id === options.turnId &&
+      end.turn.status === 'interrupted'
+    );
+  } catch {
+    return false;
+  }
+}
+
+async function runTurn(
+  options: InvocationOptions & {
+    server: Awaited<ReturnType<typeof startServer>>;
+    captured: ReturnType<typeof captureTranscript>;
+    workspace: string;
+    threadId: string;
+    signal: AbortSignal;
+  },
+) {
+  const { server, captured, threadId } = options;
+  const begin = performance.now();
+  const accepted = await server.rpc
+    .request(
+      'turn/start',
+      {
+        threadId,
+        ...knowledgeTurn,
+        input: [{ type: 'text', text: options.prompt }],
+        outputSchema: options.schema,
+      },
+      { timeoutMilliseconds: Math.min(options.deadlineMilliseconds, 30_000) },
+    )
+    .then((response) => z.looseObject({ turn: z.looseObject({ id: z.string() }) }).parse(response))
+    .catch(() => null);
+  if (!accepted) {
+    const report: InvocationReport = {
+      outcome: 'failed',
+      code: 'MODEL_START_UNCONFIRMED',
+      turnAccepted: 'unknown',
+      threadId,
+      deadlineMilliseconds: options.deadlineMilliseconds,
+      usage: null,
+    };
+    return { value: null, report, retry: false };
+  }
+  const turnId = accepted.turn.id;
+  try {
+    if (options.signal.aborted) throw new ModelCancelled();
+    const remaining = options.deadlineMilliseconds - (performance.now() - begin);
+    if (remaining <= 0) throw new ModelTimeout();
+    const end = await completedWithin({ ...options, milliseconds: remaining });
+    if (end.threadId !== threadId || end.turn.id !== turnId || end.turn.status !== 'completed')
+      throw new Error('Model completion was not established');
+    captured.assertValid({ threadId, turnId });
+    const final = captured.items.at(-1);
+    if (!final || final.threadId !== threadId || final.turnId !== turnId || !final.item.text)
+      throw new Error('Structured model output is missing');
+    if (readdirSync(options.workspace).length) throw new Error('Knowledge workspace was mutated');
+    const report: InvocationReport = {
+      outcome: 'completed',
+      threadId,
+      turnId,
+      turnAccepted: 'confirmed',
+      deadlineMilliseconds: options.deadlineMilliseconds,
+      usage: captured.measured({ threadId, turnId }),
+    };
+    return { value: final.item.text, report, retry: false };
+  } catch (error) {
+    const confirmed = await interrupt({ ...options, turnId });
+    const timeout = error instanceof ModelTimeout;
+    const cancelled = error instanceof ModelCancelled;
+    const report: InvocationReport = {
+      outcome: timeout ? 'timeout' : 'failed',
+      threadId,
+      turnId,
+      code: failureCode({ timeout, cancelled }),
+      interruption: confirmed ? 'confirmed' : 'unconfirmed',
+      turnAccepted: 'confirmed',
+      deadlineMilliseconds: options.deadlineMilliseconds,
+      usage: captured.measured({ threadId, turnId }),
+    };
+    return { value: null, report, retry: timeout && confirmed };
+  }
+}
+
+export async function invokeModel(options: InvocationOptions) {
+  const startedAt = new Date().toISOString();
+  const began = performance.now();
+  const workspace = realpathSync(mkdtempSync(join(tmpdir(), 'hivex-model-')));
+  const captured = captureTranscript();
+  const controller = new AbortController();
+  const cancel = () => controller.abort();
+  process.on('SIGINT', cancel);
+  process.on('SIGTERM', cancel);
+  const initialReport: InvocationReport = {
+    outcome: 'failed',
+    code: 'MODEL_ADMISSION_FAILED',
+    deadlineMilliseconds: options.deadlineMilliseconds,
+    usage: null,
+  };
+  const resource: {
+    server?: Awaited<ReturnType<typeof startServer>>;
+    result: { value: unknown; report: InvocationReport; retry: boolean };
+  } = { result: { value: null, report: initialReport, retry: false } };
+  try {
+    resource.server = await startServer({
+      binary: options.binary,
+      workspace,
+      signal: controller.signal,
+      notification: captured.notification,
+      interaction: () => {
+        throw new Error('Knowledge execution cannot request interaction');
+      },
+    });
+    options.onNativeProcessStarted?.(resource.server.pid);
+    const threadId = await startKnowledgeThread({
+      rpc: resource.server.rpc,
+      workspace,
+      signal: controller.signal,
+    });
+    controller.signal.throwIfAborted();
+    resource.result = await runTurn({
+      ...options,
+      captured,
+      workspace,
+      server: resource.server,
+      threadId,
+      signal: controller.signal,
+    });
+  } catch (error) {
+    if (controller.signal.aborted) initialReport.code = 'MODEL_CANCELLED';
+    initialReport.diagnostic = failureDiagnostic(error);
+    if (error instanceof ServerAdmissionFailure) {
+      initialReport.cleanup = error.cleanup;
+      initialReport.nativeProcessId = error.processId;
+      initialReport.admission = error.admission;
+    }
+    resource.result = { value: null, report: initialReport, retry: false };
+  } finally {
+    try {
+      await resource.server?.stop();
+      rmSync(workspace, { recursive: true, force: true });
+      resource.result.report.cleanup = resource.server
+        ? 'confirmed'
+        : (resource.result.report.cleanup ?? 'not-observed');
+    } catch {
+      resource.result = {
+        value: null,
+        retry: false,
+        report: {
+          ...resource.result.report,
+          outcome: 'failed',
+          code: 'MODEL_CLEANUP_FAILED',
+          cleanup: 'failed',
+        },
+      };
+    }
+    process.off('SIGINT', cancel);
+    process.off('SIGTERM', cancel);
+  }
+  const report = resource.result.report;
+  if (report.threadId && report.turnId)
+    report.usage = captured.measured({ threadId: report.threadId, turnId: report.turnId });
+  if (captured.invalid && report.outcome === 'completed') {
+    resource.result = {
+      value: null,
+      retry: false,
+      report: { ...report, outcome: 'failed', code: 'MODEL_PROTOCOL_FAILED' },
+    };
+  }
+  return {
+    ...resource.result,
+    report: {
+      ...resource.result.report,
+      admission: resource.server?.admission ?? resource.result.report.admission,
+      nativeProcessId: resource.server?.pid ?? resource.result.report.nativeProcessId,
+      startedAt,
+      durationMilliseconds: Math.round(performance.now() - began),
+    },
+  };
+}
+
+function failureCode(options: { timeout: boolean; cancelled: boolean }) {
+  if (options.cancelled) return 'MODEL_CANCELLED';
+  return options.timeout ? 'MODEL_TIMEOUT' : 'MODEL_PROTOCOL_FAILED';
+}