ds4-context-engine 0.1.0

package/src/pi-adapter/session-jsonl.ts

@@ -0,0 +1,183 @@
+import { closeSync, existsSync, fstatSync, openSync, readSync } from "node:fs";
+import { sha256 } from "ds4-context-core/shared/hash";
+
+const READ_BUFFER_SIZE = 1024 * 1024;
+const MAX_HEADER_BYTES = 1024 * 1024;
+const MAX_CHECKPOINT_HASH_BYTES = 64 * 1024;
+
+export interface JsonlRecord {
+  value: Record<string, unknown>;
+  raw: string;
+  rawHash: string;
+  startOffset: number;
+  endOffset: number;
+  terminated: boolean;
+}
+
+export interface SessionHeaderRecord extends JsonlRecord {
+  value: Record<string, unknown> & { type: "session"; id: string };
+}
+
+export interface JsonlReadResult {
+  records: JsonlRecord[];
+  malformedLines: number;
+  fileSize: number;
+  fileMtimeMs: number;
+  safeCheckpointOffset: number;
+  checkpointHashStart: number;
+  checkpointHash?: string;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function parseLine(
+  bytes: Buffer,
+  startOffset: number,
+  endOffset: number,
+  terminated: boolean,
+): JsonlRecord | "blank" | "malformed" {
+  let parseBytes = bytes;
+  if (parseBytes.at(-1) === 0x0d) parseBytes = parseBytes.subarray(0, -1);
+  const raw = parseBytes.toString("utf8");
+  if (raw.trim().length === 0) return "blank";
+
+  try {
+    const value: unknown = JSON.parse(raw);
+    if (!isRecord(value) || typeof value.type !== "string") return "malformed";
+    return {
+      value,
+      raw,
+      rawHash: sha256(bytes),
+      startOffset,
+      endOffset,
+      terminated,
+    };
+  } catch {
+    return "malformed";
+  }
+}
+
+export function readSessionHeaderRecord(filePath: string): SessionHeaderRecord {
+  if (!existsSync(filePath)) throw new Error(`Session file does not exist: ${filePath}`);
+
+  const fd = openSync(filePath, "r");
+  try {
+    const stat = fstatSync(fd);
+    const readLength = Math.min(stat.size, MAX_HEADER_BYTES);
+    const buffer = Buffer.allocUnsafe(readLength);
+    const bytesRead = readLength > 0 ? readSync(fd, buffer, 0, readLength, 0) : 0;
+    const data = buffer.subarray(0, bytesRead);
+    let lineStart = 0;
+
+    for (let index = 0; index <= data.length; index++) {
+      const atNewline = index < data.length && data[index] === 0x0a;
+      const atCompleteEof = index === data.length && stat.size <= MAX_HEADER_BYTES;
+      if (!atNewline && !atCompleteEof) continue;
+
+      const line = data.subarray(lineStart, index);
+      const parsed = parseLine(line, lineStart, atNewline ? index + 1 : index, atNewline);
+      lineStart = index + 1;
+      if (parsed === "blank" || parsed === "malformed") continue;
+      if (parsed.value.type !== "session" || typeof parsed.value.id !== "string") {
+        throw new Error(`First parsed JSONL record is not a Pi session header: ${filePath}`);
+      }
+      return parsed as SessionHeaderRecord;
+    }
+
+    throw new Error(`No Pi session header found within ${MAX_HEADER_BYTES} bytes: ${filePath}`);
+  } finally {
+    closeSync(fd);
+  }
+}
+
+export function readJsonlRecords(filePath: string, startOffset = 0): JsonlReadResult {
+  if (!Number.isSafeInteger(startOffset) || startOffset < 0) {
+    throw new Error("JSONL start offset must be a non-negative safe integer");
+  }
+
+  const fd = openSync(filePath, "r");
+  try {
+    const stat = fstatSync(fd);
+    if (startOffset > stat.size) throw new Error("JSONL start offset is beyond end of file");
+
+    const records: JsonlRecord[] = [];
+    let malformedLines = 0;
+    let safeCheckpointOffset = startOffset;
+    let checkpointHashStart = startOffset;
+    let checkpointHash: string | undefined;
+    let pending = Buffer.alloc(0);
+    let pendingStart = startOffset;
+    let position = startOffset;
+
+    while (position < stat.size) {
+      const length = Math.min(READ_BUFFER_SIZE, stat.size - position);
+      const chunk = Buffer.allocUnsafe(length);
+      const bytesRead = readSync(fd, chunk, 0, length, position);
+      if (bytesRead === 0) break;
+      position += bytesRead;
+
+      const combined = pending.length === 0
+        ? chunk.subarray(0, bytesRead)
+        : Buffer.concat([pending, chunk.subarray(0, bytesRead)]);
+      let lineStart = 0;
+
+      for (let index = 0; index < combined.length; index++) {
+        if (combined[index] !== 0x0a) continue;
+
+        const absoluteStart = pendingStart + lineStart;
+        const absoluteEnd = pendingStart + index + 1;
+        const line = combined.subarray(lineStart, index);
+        const parsed = parseLine(line, absoluteStart, absoluteEnd, true);
+        if (parsed === "malformed") malformedLines++;
+        else if (parsed !== "blank") records.push(parsed);
+
+        safeCheckpointOffset = absoluteEnd;
+        const boundedHashStart = Math.max(lineStart, index + 1 - MAX_CHECKPOINT_HASH_BYTES);
+        checkpointHashStart = pendingStart + boundedHashStart;
+        checkpointHash = sha256(combined.subarray(boundedHashStart, index + 1));
+        lineStart = index + 1;
+      }
+
+      pending = Buffer.from(combined.subarray(lineStart));
+      pendingStart += lineStart;
+    }
+
+    if (pending.length > 0) {
+      const parsed = parseLine(pending, pendingStart, stat.size, false);
+      if (parsed === "malformed") malformedLines++;
+      else if (parsed !== "blank") records.push(parsed);
+    }
+
+    return {
+      records,
+      malformedLines,
+      fileSize: stat.size,
+      fileMtimeMs: stat.mtimeMs,
+      safeCheckpointOffset,
+      checkpointHashStart,
+      ...(checkpointHash ? { checkpointHash } : {}),
+    };
+  } finally {
+    closeSync(fd);
+  }
+}
+
+export function hashFileRange(filePath: string, startOffset: number, endOffset: number): string {
+  if (startOffset < 0 || endOffset < startOffset) throw new Error("Invalid file range");
+  const length = endOffset - startOffset;
+  const fd = openSync(filePath, "r");
+  try {
+    const buffer = Buffer.allocUnsafe(length);
+    let total = 0;
+    while (total < length) {
+      const bytesRead = readSync(fd, buffer, total, length - total, startOffset + total);
+      if (bytesRead === 0) throw new Error("Unexpected end of file while validating index checkpoint");
+      total += bytesRead;
+    }
+    return sha256(buffer);
+  } finally {
+    closeSync(fd);
+  }
+}

package/src/pi-adapter/session-reader.ts

@@ -0,0 +1,39 @@
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+import type { ModelDescriptor } from "ds4-context-core/core/model-profile";
+
+export interface PiSessionSnapshot {
+  sessionId: string;
+  sessionFile?: string;
+  projectPath: string;
+  leafId?: string;
+  totalEntries: number;
+  branchEntries: number;
+}
+
+type SessionContext = Pick<ExtensionContext, "cwd" | "sessionManager">;
+
+export function snapshotSession(ctx: SessionContext): PiSessionSnapshot {
+  const sessionFile = ctx.sessionManager.getSessionFile();
+  const leafId = ctx.sessionManager.getLeafId();
+
+  return {
+    sessionId: ctx.sessionManager.getSessionId(),
+    ...(sessionFile ? { sessionFile } : {}),
+    projectPath: ctx.cwd,
+    ...(leafId ? { leafId } : {}),
+    totalEntries: ctx.sessionManager.getEntries().length,
+    branchEntries: ctx.sessionManager.getBranch().length,
+  };
+}
+
+export function snapshotModel(ctx: Pick<ExtensionContext, "model">): ModelDescriptor | undefined {
+  if (!ctx.model) return undefined;
+  return {
+    provider: ctx.model.provider,
+    id: ctx.model.id,
+    contextWindow: ctx.model.contextWindow,
+    maxTokens: ctx.model.maxTokens,
+    reasoning: ctx.model.reasoning,
+    input: ctx.model.input,
+  };
+}

package/src/pi-adapter/summary-generator.ts

@@ -0,0 +1,157 @@
+import { randomUUID } from "node:crypto";
+import type { Usage } from "@earendil-works/pi-ai";
+import type {
+  ExtensionContext,
+  SessionBeforeCompactEvent,
+} from "@earendil-works/pi-coding-agent";
+import {
+  groundSummaryFileSections,
+  pruneUnsupportedExactValueBullets,
+  validateSummary,
+  type SummaryValidationInput,
+  type SummaryValidationResult,
+} from "ds4-context-core/compaction/summary-contract";
+
+export interface GenerateValidatedSummaryInput {
+  stage: "segment" | "aggregate";
+  prompt: string;
+  validationSource: string;
+  readFiles: readonly string[];
+  modifiedFiles: readonly string[];
+  validate: boolean;
+  maxSummaryTokens: number;
+  event: SessionBeforeCompactEvent;
+  ctx: ExtensionContext;
+  now: () => number;
+}
+
+export interface GeneratedSummary {
+  content: string;
+  validation: SummaryValidationResult;
+  usage: Usage;
+}
+
+function unique(values: readonly string[]): string[] {
+  return [...new Set(values)];
+}
+
+function responseText(response: unknown): string {
+  if (!response || typeof response !== "object" || !("content" in response)) return "";
+  const content = (response as { content?: unknown }).content;
+  if (!Array.isArray(content)) return "";
+  return content.flatMap((block) => {
+    if (!block || typeof block !== "object" || !("type" in block) || !("text" in block)) return [];
+    return block.type === "text" && typeof block.text === "string" ? [block.text] : [];
+  }).join("\n").trim();
+}
+
+function responseStopReason(response: unknown): string | undefined {
+  if (!response || typeof response !== "object" || !("stopReason" in response)) return undefined;
+  return typeof response.stopReason === "string" ? response.stopReason : undefined;
+}
+
+export async function generateValidatedSummary(
+  input: GenerateValidatedSummaryInput,
+): Promise<GeneratedSummary> {
+  const model = input.ctx.model;
+  if (!model) throw new Error("Compaction summary generation requires an active model");
+  const maxTokens = Math.max(1, Math.min(input.maxSummaryTokens, model.maxTokens ?? input.maxSummaryTokens));
+  const response = await input.ctx.modelRegistry.complete(
+    model,
+    {
+      messages: [{
+        role: "user",
+        content: [{ type: "text", text: input.prompt }],
+        timestamp: input.now(),
+      }],
+    },
+    {
+      maxTokens,
+      signal: input.event.signal,
+      cacheRetention: "none",
+      sessionId: randomUUID(),
+    },
+  );
+  if (input.event.signal.aborted) throw new Error("Compaction summary generation aborted");
+  const stopReason = responseStopReason(response);
+  if (stopReason === "length") throw new Error("Compaction summary hit the model output limit");
+  if (stopReason === "error" || stopReason === "aborted") {
+    throw new Error(`Compaction summary stopped with ${stopReason}`);
+  }
+  if (response.content.some((block) => block.type === "toolCall")) {
+    throw new Error("Compaction summarizer attempted to call a tool");
+  }
+  const generatedContent = responseText(response);
+  if (!generatedContent) throw new Error("Compaction summarizer returned empty text");
+  let content = groundSummaryFileSections(generatedContent, {
+    readFiles: input.readFiles,
+    modifiedFiles: input.modifiedFiles,
+  });
+  const validationInput: SummaryValidationInput = {
+    sourceText: input.validationSource,
+    readFiles: input.readFiles,
+    modifiedFiles: input.modifiedFiles,
+  };
+  let validation: SummaryValidationResult = input.validate
+    ? validateSummary(content, validationInput)
+    : {
+        status: "warning",
+        issues: [{
+          code: "validation-disabled",
+          severity: "warning",
+          message: "Deterministic validation disabled by configuration",
+        }],
+      };
+  if (validation.status === "invalid") {
+    const errors = validation.issues.filter((issue) => issue.severity === "error");
+    const exactOnly = errors.length > 0
+      && errors.every((issue) => issue.code === "unsupported-exact-value");
+    const pruned = exactOnly
+      ? pruneUnsupportedExactValueBullets(content, validationInput)
+      : undefined;
+    if (pruned) {
+      const repairedValidation = validateSummary(pruned.content, validationInput);
+      if (repairedValidation.status !== "invalid") {
+        content = pruned.content;
+        validation = {
+          status: "warning",
+          issues: [
+            ...repairedValidation.issues,
+            {
+              code: "unsupported-exact-bullets-pruned",
+              severity: "warning",
+              message: `Removed ${pruned.removedBullets} bullet(s) containing unsupported exact values`,
+            },
+          ],
+        };
+      }
+    }
+  }
+  if (validation.status === "invalid") {
+    const codes = unique(validation.issues.map((issue) => issue.code)).join(", ");
+    throw new Error(`Compaction ${input.stage} summary validation failed: ${codes}`);
+  }
+  return { content, validation, usage: response.usage };
+}
+
+export function sumUsage(usages: readonly Usage[]): Usage {
+  const sum = (read: (usage: Usage) => number): number => usages.reduce((total, usage) => total + read(usage), 0);
+  const hasReasoning = usages.some((usage) => usage.reasoning !== undefined);
+  const hasCacheWrite1h = usages.some((usage) => usage.cacheWrite1h !== undefined);
+  return {
+    input: sum((usage) => usage.input),
+    output: sum((usage) => usage.output),
+    cacheRead: sum((usage) => usage.cacheRead),
+    cacheWrite: sum((usage) => usage.cacheWrite),
+    ...(hasCacheWrite1h ? { cacheWrite1h: sum((usage) => usage.cacheWrite1h ?? 0) } : {}),
+    ...(hasReasoning ? { reasoning: sum((usage) => usage.reasoning ?? 0) } : {}),
+    totalTokens: sum((usage) => usage.totalTokens),
+    cost: {
+      input: sum((usage) => usage.cost.input),
+      output: sum((usage) => usage.cost.output),
+      cacheRead: sum((usage) => usage.cost.cacheRead),
+      cacheWrite: sum((usage) => usage.cost.cacheWrite),
+      total: sum((usage) => usage.cost.total),
+    },
+  };
+}

package/src/pi-adapter/version.ts

@@ -0,0 +1,5 @@
+export const EXTENSION_VERSION = "0.1.0";
+export const SUPPORTED_PI_VERSION = "0.84.3";
+export const OBSERVER_PLANNER_VERSION = "observer-model-aware-v1";
+export const PLANNER_VERSION = "managed-native-continuation-v1";
+export const POLICY_VERSION = "8";