harnery 0.34.1 → 0.36.0

package/src/core/conversations/catalog.ts

@@ -0,0 +1,79 @@
+import type { HarneryConversationProvider } from "./contract.ts";
+
+export class HarneryConversationCatalog {
+  readonly #providers: ReadonlyMap<string, HarneryConversationProvider>;
+
+  constructor(providers: readonly HarneryConversationProvider[]) {
+    const entries = new Map<string, HarneryConversationProvider>();
+    for (const provider of providers) {
+      const id = provider.capabilities.provider_id;
+      if (!/^[a-z][a-z0-9-]{1,63}$/.test(id))
+        throw new Error(`invalid conversation provider id: ${id}`);
+      if (entries.has(id)) throw new Error(`duplicate conversation provider: ${id}`);
+      if (provider.capabilities.roles.some((role) => role !== "user" && role !== "assistant")) {
+        throw new Error(`provider ${id} exposes an excluded role`);
+      }
+      entries.set(id, Object.freeze(provider));
+    }
+    this.#providers = entries;
+  }
+
+  get(id: string): HarneryConversationProvider | undefined {
+    return this.#providers.get(id);
+  }
+
+  require(id: string): HarneryConversationProvider {
+    const provider = this.get(id);
+    if (!provider) throw new Error(`unknown conversation provider: ${id}`);
+    return provider;
+  }
+
+  list(): readonly HarneryConversationProvider[] {
+    return [...this.#providers.values()].sort((left, right) =>
+      left.capabilities.provider_id.localeCompare(right.capabilities.provider_id),
+    );
+  }
+}
+
+export function unavailableAdapterProviders(): readonly HarneryConversationProvider[] {
+  return ["claude-code", "codex", "cursor"].map((providerId) => ({
+    capabilities: {
+      provider_id: providerId,
+      roles: ["user", "assistant"] as const,
+      can_list: false,
+      can_stream_source: false,
+      can_replay_archive: false,
+      default_completeness: "unavailable" as const,
+      default_omissions: [
+        "adapter-native source reader is not registered by this host",
+        "system, developer, tool, reasoning, command, and attachment bodies are excluded",
+      ],
+      retention_behavior: "adapter-owned and unavailable to portable core",
+    },
+    async list() {
+      return [];
+    },
+    async snapshot(projectScopeId: string, conversationId: string) {
+      return {
+        snapshot_id: `${providerId}:unavailable:${conversationId}`,
+        provider_id: providerId,
+        project_scope_id: projectScopeId,
+        conversation_id: conversationId,
+        observed_at: new Date().toISOString(),
+        completeness: "unavailable" as const,
+        omissions: ["adapter-native source unavailable"],
+      };
+    },
+    stream() {
+      return {
+        [Symbol.asyncIterator]() {
+          return {
+            async next(): Promise<IteratorResult<never>> {
+              throw new Error(`conversation provider unavailable: ${providerId}`);
+            },
+          };
+        },
+      };
+    },
+  }));
+}

package/src/core/conversations/context-pack.ts

@@ -0,0 +1,72 @@
+import type { HarneryConversationContextPack, HarneryConversationQueryResult } from "./contract.ts";
+
+export function buildConversationContextPack(
+  result: HarneryConversationQueryResult,
+  budget: { max_tokens: number; max_bytes: number },
+): HarneryConversationContextPack {
+  if (
+    !Number.isSafeInteger(budget.max_tokens) ||
+    budget.max_tokens <= 0 ||
+    !Number.isSafeInteger(budget.max_bytes) ||
+    budget.max_bytes <= 0
+  ) {
+    throw new Error("invalid conversation context budget");
+  }
+  const ordered = new Map(
+    result.hits
+      .flatMap((hit) => [...hit.neighbors, hit.record])
+      .sort(
+        (left, right) =>
+          left.provider_id.localeCompare(right.provider_id) ||
+          left.conversation_id.localeCompare(right.conversation_id) ||
+          left.sequence - right.sequence,
+      )
+      .map((record) => [record.record_id, record]),
+  );
+  const excerpts: HarneryConversationContextPack["excerpts"][number][] = [];
+  let bytes = 0;
+  let tokens = 0;
+  let truncated = result.truncated;
+  for (const record of ordered.values()) {
+    const recordBytes = Buffer.byteLength(record.content);
+    const recordTokens = Math.ceil(recordBytes / 4);
+    if (bytes + recordBytes > budget.max_bytes || tokens + recordTokens > budget.max_tokens) {
+      truncated = true;
+      break;
+    }
+    excerpts.push({
+      citation: {
+        record_id: record.record_id,
+        provider_id: record.provider_id,
+        conversation_id: record.conversation_id,
+        role: record.role,
+        occurred_at: record.occurred_at,
+        native_conversation_id: record.source.native_conversation_id,
+        ...(record.source.native_record_id
+          ? { native_record_id: record.source.native_record_id }
+          : {}),
+        content_digest: record.content_digest,
+      },
+      content: record.content,
+      sequence: record.sequence,
+    });
+    bytes += recordBytes;
+    tokens += recordTokens;
+  }
+  return {
+    schema: "harnery.conversation-context-pack/v1",
+    boundary: "untrusted-historical-data",
+    automatic_injection: false,
+    project_scope_id: result.request.project_scope_id,
+    excerpts,
+    completeness: result.snapshots.map((snapshot) => ({
+      provider_id: snapshot.provider_id,
+      conversation_id: snapshot.conversation_id,
+      completeness: snapshot.completeness,
+      omissions: snapshot.omissions,
+    })),
+    bytes,
+    estimated_tokens: tokens,
+    truncated,
+  };
+}

package/src/core/conversations/contract.ts

@@ -0,0 +1,178 @@
+export type HarneryConversationRole = "user" | "assistant";
+export type HarneryConversationCompleteness = "complete" | "partial" | "unavailable";
+export type HarneryConversationAccessMode = "off" | "source" | "archive";
+export type HarneryConversationArchiveAuthorityMode = "shadow" | "cutover" | "rollback";
+
+export interface HarneryConversationProviderCapabilities {
+  provider_id: string;
+  roles: readonly HarneryConversationRole[];
+  can_list: boolean;
+  can_stream_source: boolean;
+  can_replay_archive: boolean;
+  default_completeness: HarneryConversationCompleteness;
+  default_omissions: readonly string[];
+  retention_behavior: string;
+}
+
+export interface HarneryConversationSourceSnapshot {
+  snapshot_id: string;
+  provider_id: string;
+  project_scope_id: string;
+  conversation_id: string;
+  checkpoint?: string;
+  observed_at: string;
+  completeness: HarneryConversationCompleteness;
+  omissions: readonly string[];
+}
+
+export interface HarneryConversationRecordV1 {
+  schema: "harnery.conversation-record/v1";
+  record_id: string;
+  provider_id: string;
+  project_scope_id: string;
+  conversation_id: string;
+  source_snapshot_id: string;
+  session_id?: string;
+  turn_id?: string;
+  source: {
+    native_conversation_id: string;
+    native_record_id?: string;
+    native_sequence?: number;
+    source_digest?: `sha256:${string}`;
+  };
+  sequence: number;
+  occurred_at: string;
+  captured_at: string;
+  role: HarneryConversationRole;
+  content: string;
+  content_bytes: number;
+  content_digest: `sha256:${string}`;
+  redactions: readonly {
+    kind: string;
+    replacement: string;
+    stored_start: number;
+    stored_end: number;
+    source_bytes_removed: number;
+  }[];
+  attachment_refs: readonly string[];
+  sensitivity: "private";
+}
+
+export interface HarneryConversationSummary {
+  provider_id: string;
+  project_scope_id: string;
+  conversation_id: string;
+  snapshot_id: string;
+  completeness: HarneryConversationCompleteness;
+  omissions: readonly string[];
+  record_count?: number;
+}
+
+export interface HarneryConversationCitation {
+  record_id: string;
+  provider_id: string;
+  conversation_id: string;
+  role: HarneryConversationRole;
+  occurred_at: string;
+  native_conversation_id: string;
+  native_record_id?: string;
+  content_digest: `sha256:${string}`;
+}
+
+export interface HarneryConversationQueryRequest {
+  text?: string;
+  regex?: string;
+  provider_id?: string;
+  role?: HarneryConversationRole;
+  since?: string;
+  until?: string;
+  project_scope_id: string;
+  session_id?: string;
+  conversation_id?: string;
+  record_id?: string;
+  limit: number;
+  context_before?: number;
+  context_after?: number;
+  budgets: HarneryConversationQueryBudgets;
+}
+
+export interface HarneryConversationQueryBudgets {
+  max_source_records: number;
+  max_decoded_bytes: number;
+  max_matches: number;
+  max_wall_ms: number;
+  max_regex_chars: number;
+}
+
+export interface HarneryConversationQueryHit {
+  record: HarneryConversationRecordV1;
+  citation: HarneryConversationCitation;
+  neighbors: readonly HarneryConversationRecordV1[];
+}
+
+export interface HarneryConversationQueryResult {
+  schema: "harnery.conversation-query/v1";
+  request: HarneryConversationQueryRequest;
+  hits: readonly HarneryConversationQueryHit[];
+  scanned_records: number;
+  decoded_bytes: number;
+  truncated: boolean;
+  truncation_reason?: "record_budget" | "byte_budget" | "match_budget" | "wall_budget";
+  snapshots: readonly HarneryConversationSourceSnapshot[];
+}
+
+export interface HarneryConversationContextPack {
+  schema: "harnery.conversation-context-pack/v1";
+  boundary: "untrusted-historical-data";
+  automatic_injection: false;
+  project_scope_id: string;
+  excerpts: readonly {
+    citation: HarneryConversationCitation;
+    content: string;
+    sequence: number;
+  }[];
+  completeness: readonly {
+    provider_id: string;
+    conversation_id: string;
+    completeness: HarneryConversationCompleteness;
+    omissions: readonly string[];
+  }[];
+  bytes: number;
+  estimated_tokens: number;
+  truncated: boolean;
+}
+
+export interface HarneryNativeConversationRecord {
+  native_conversation_id: string;
+  native_record_id?: string;
+  native_sequence?: number;
+  role: string;
+  occurred_at: string;
+  content: string;
+  session_id?: string;
+  turn_id?: string;
+  attachment_refs?: readonly string[];
+}
+
+export interface HarneryConversationProviderReadOptions {
+  /** Providers should stop pending I/O promptly when the query wall budget expires. */
+  signal?: AbortSignal;
+}
+
+export interface HarneryConversationProvider {
+  capabilities: HarneryConversationProviderCapabilities;
+  list(
+    projectScopeId: string,
+    options?: HarneryConversationProviderReadOptions,
+  ): Promise<readonly HarneryConversationSummary[]>;
+  snapshot(
+    projectScopeId: string,
+    conversationId: string,
+    options?: HarneryConversationProviderReadOptions,
+  ): Promise<HarneryConversationSourceSnapshot>;
+  stream(
+    projectScopeId: string,
+    conversationId: string,
+    options?: HarneryConversationProviderReadOptions,
+  ): AsyncIterable<HarneryNativeConversationRecord>;
+}

package/src/core/conversations/evaluation.ts

@@ -0,0 +1,30 @@
+export interface HarneryConversationEvaluationResult {
+  fixture_count: number;
+  citation_accuracy: number;
+  privacy_failures: number;
+  instruction_following_from_history: number;
+  recall: number;
+  precision: number;
+}
+
+export interface HarneryConversationEvaluationThresholds {
+  min_fixture_count: number;
+  min_citation_accuracy: number;
+  min_recall: number;
+  min_precision: number;
+}
+
+export function automaticConversationInjectionEnabled(
+  result: HarneryConversationEvaluationResult | undefined,
+  thresholds: HarneryConversationEvaluationThresholds,
+): boolean {
+  return Boolean(
+    result &&
+      result.fixture_count >= thresholds.min_fixture_count &&
+      result.citation_accuracy >= thresholds.min_citation_accuracy &&
+      result.recall >= thresholds.min_recall &&
+      result.precision >= thresholds.min_precision &&
+      result.privacy_failures === 0 &&
+      result.instruction_following_from_history === 0,
+  );
+}

package/src/core/conversations/index.ts

@@ -0,0 +1,9 @@
+export * from "./archive.ts";
+export * from "./catalog.ts";
+export * from "./context-pack.ts";
+export * from "./contract.ts";
+export * from "./evaluation.ts";
+export * from "./lifecycle.ts";
+export * from "./normalize.ts";
+export * from "./query.ts";
+export * from "./search-index.ts";

package/src/core/conversations/lifecycle.ts

@@ -0,0 +1,63 @@
+import { randomUUID } from "node:crypto";
+import type { HarneryConversationRecordV1 } from "./contract.ts";
+
+export const HARNERY_CONVERSATION_PURGE_PLAN_SCHEMA = "harnery.conversation-purge-plan/v1" as const;
+
+export interface HarneryConversationPurgePlan {
+  schema: typeof HARNERY_CONVERSATION_PURGE_PLAN_SCHEMA;
+  transaction_id: string;
+  project_scope_id: string;
+  conversation_id?: string;
+  before?: string;
+  record_ids: readonly string[];
+  record_digests: readonly string[];
+  remove_bytes: number;
+  retained_record_count: number;
+  external_source_state: "adapter-owned-copy-may-remain";
+  dry_run: true;
+  executable: false;
+}
+
+export function planConversationPurge(input: {
+  records: readonly HarneryConversationRecordV1[];
+  project_scope_id: string;
+  conversation_id?: string;
+  before?: string;
+  transaction_id?: string;
+}): HarneryConversationPurgePlan {
+  const selected = input.records.filter(
+    (record) =>
+      record.project_scope_id === input.project_scope_id &&
+      (!input.conversation_id || record.conversation_id === input.conversation_id) &&
+      (!input.before || record.occurred_at < new Date(input.before).toISOString()),
+  );
+  return {
+    schema: HARNERY_CONVERSATION_PURGE_PLAN_SCHEMA,
+    transaction_id: input.transaction_id ?? `cpurge_${randomUUID()}`,
+    project_scope_id: input.project_scope_id,
+    ...(input.conversation_id ? { conversation_id: input.conversation_id } : {}),
+    ...(input.before ? { before: new Date(input.before).toISOString() } : {}),
+    record_ids: selected.map(({ record_id }) => record_id).sort(),
+    record_digests: selected.map(({ content_digest }) => content_digest).sort(),
+    remove_bytes: selected.reduce((sum, record) => sum + record.content_bytes, 0),
+    retained_record_count: input.records.length - selected.length,
+    external_source_state: "adapter-owned-copy-may-remain",
+    dry_run: true,
+    executable: false,
+  };
+}
+
+export function validatePurgeExecutionRequest(
+  plan: HarneryConversationPurgePlan,
+  transactionId: string | undefined,
+  yes: boolean,
+): {
+  accepted: false;
+  reason_code: "confirmation_required" | "transaction_mismatch" | "execution_disabled";
+} {
+  if (!yes) return { accepted: false, reason_code: "confirmation_required" };
+  if (transactionId !== plan.transaction_id) {
+    return { accepted: false, reason_code: "transaction_mismatch" };
+  }
+  return { accepted: false, reason_code: "execution_disabled" };
+}

package/src/core/conversations/normalize.ts

@@ -0,0 +1,105 @@
+import { createHash } from "node:crypto";
+import type {
+  HarneryConversationProviderCapabilities,
+  HarneryConversationRecordV1,
+  HarneryConversationSourceSnapshot,
+  HarneryNativeConversationRecord,
+} from "./contract.ts";
+
+export interface NormalizeConversationResult {
+  record?: HarneryConversationRecordV1;
+  omission?: string;
+}
+
+export function normalizeConversationRecord(input: {
+  capabilities: HarneryConversationProviderCapabilities;
+  snapshot: HarneryConversationSourceSnapshot;
+  native: HarneryNativeConversationRecord;
+  captured_at: string;
+  sequence: number;
+}): NormalizeConversationResult {
+  if (input.native.role !== "user" && input.native.role !== "assistant") {
+    return { omission: `excluded_role:${input.native.role}` };
+  }
+  if (!input.capabilities.roles.includes(input.native.role)) {
+    return { omission: `unsupported_role:${input.native.role}` };
+  }
+  const sourceContent = input.native.content;
+  const sourceDigest = digest(sourceContent);
+  const redacted = redact(sourceContent);
+  const content = redacted.content;
+  const contentDigest = digest(content);
+  const nativeLocator = [
+    input.capabilities.provider_id,
+    input.native.native_conversation_id,
+    input.native.native_record_id ?? "",
+    String(input.native.native_sequence ?? ""),
+    input.native.role,
+    input.native.occurred_at,
+    sourceDigest,
+  ].join("\0");
+  return {
+    record: {
+      schema: "harnery.conversation-record/v1",
+      record_id: `cr_${digest(nativeLocator).slice("sha256:".length, 38)}`,
+      provider_id: input.capabilities.provider_id,
+      project_scope_id: input.snapshot.project_scope_id,
+      conversation_id: input.snapshot.conversation_id,
+      source_snapshot_id: input.snapshot.snapshot_id,
+      ...(input.native.session_id ? { session_id: input.native.session_id } : {}),
+      ...(input.native.turn_id ? { turn_id: input.native.turn_id } : {}),
+      source: {
+        native_conversation_id: input.native.native_conversation_id,
+        ...(input.native.native_record_id
+          ? { native_record_id: input.native.native_record_id }
+          : {}),
+        ...(input.native.native_sequence !== undefined
+          ? { native_sequence: input.native.native_sequence }
+          : {}),
+        source_digest: sourceDigest,
+      },
+      sequence: input.sequence,
+      occurred_at: new Date(input.native.occurred_at).toISOString(),
+      captured_at: new Date(input.captured_at).toISOString(),
+      role: input.native.role,
+      content,
+      content_bytes: Buffer.byteLength(content),
+      content_digest: contentDigest,
+      redactions: redacted.redactions,
+      attachment_refs: [...(input.native.attachment_refs ?? [])],
+      sensitivity: "private",
+    },
+  };
+}
+
+function redact(content: string): {
+  content: string;
+  redactions: HarneryConversationRecordV1["redactions"];
+} {
+  const pattern = /\b(?:sk|key|token|secret)[-_][a-zA-Z0-9_-]{8,}\b/giu;
+  const redactions: HarneryConversationRecordV1["redactions"][number][] = [];
+  let stored = "";
+  let sourceOffset = 0;
+  for (const match of content.matchAll(pattern)) {
+    const start = match.index;
+    const value = match[0];
+    stored += content.slice(sourceOffset, start);
+    const replacement = "[REDACTED:secret]";
+    const storedStart = stored.length;
+    stored += replacement;
+    redactions.push({
+      kind: "secret",
+      replacement,
+      stored_start: storedStart,
+      stored_end: storedStart + replacement.length,
+      source_bytes_removed: Buffer.byteLength(value),
+    });
+    sourceOffset = start + value.length;
+  }
+  stored += content.slice(sourceOffset);
+  return { content: stored, redactions };
+}
+
+export function digest(value: string | Buffer): `sha256:${string}` {
+  return `sha256:${createHash("sha256").update(value).digest("hex")}`;
+}