harnery 0.34.1 → 0.36.0

package/src/core/storage/query.ts

@@ -0,0 +1,360 @@
+import {
+  closeSync,
+  existsSync,
+  constants as fsConstants,
+  fstatSync,
+  lstatSync,
+  openSync,
+  readSync,
+  realpathSync,
+} from "node:fs";
+import { dirname, isAbsolute, join, relative } from "node:path";
+import { Readable } from "node:stream";
+import { createGunzip } from "node:zlib";
+import type { HarneryLogLevel, HarneryRegisteredStorageFamily } from "./contract.ts";
+import type { HarneryLogRecordV1 } from "./jsonl.ts";
+import { parseLogRecord } from "./jsonl.ts";
+import { familyLogDirectory, readSegmentManifest } from "./segments.ts";
+
+export interface HarneryLogQuery {
+  family_ids?: readonly string[];
+  minimum_level?: HarneryLogLevel;
+  event?: string;
+  since?: string;
+  until?: string;
+  context?: Readonly<Record<string, string | number | boolean | null>>;
+  max_records: number;
+  max_bytes: number;
+}
+
+export interface HarneryLogQueryResult {
+  records: readonly HarneryLogRecordV1[];
+  records_examined: number;
+  bytes_examined: number;
+  truncated: boolean;
+  /** Per-family highest sequence intentionally expired by retention. */
+  expired_through: Readonly<Record<string, number>>;
+}
+
+export interface HarneryLogFollowCursor {
+  family_id: string;
+  manifest_sequence: number;
+  active_offset: number;
+}
+
+const LEVELS: readonly HarneryLogLevel[] = ["trace", "debug", "info", "warn", "error", "fatal"];
+const MAX_UNBOUNDED_RECORD_BYTES = 1024 * 1024;
+const STREAM_CHUNK_BYTES = 64 * 1024;
+
+export async function queryLogs(
+  families: readonly HarneryRegisteredStorageFamily[],
+  query: HarneryLogQuery,
+): Promise<HarneryLogQueryResult> {
+  if (
+    !Number.isSafeInteger(query.max_records) ||
+    query.max_records <= 0 ||
+    !Number.isSafeInteger(query.max_bytes) ||
+    query.max_bytes <= 0
+  )
+    throw new Error("log query requires positive global budgets");
+  const records: HarneryLogRecordV1[] = [];
+  let bytes = 0;
+  let examined = 0;
+  let truncated = false;
+  const expiredThrough: Record<string, number> = {};
+  for (const family of families) {
+    if (query.family_ids && !query.family_ids.includes(family.id)) continue;
+    const directory = familyLogDirectory(family);
+    const manifest = readSegmentManifest(directory, family);
+    expiredThrough[family.id] = manifest.pruned_through_sequence ?? 0;
+    const sources = manifestSources(directory, manifest);
+    const maximumLineBytes =
+      family.policy.records.max_record_bytes.limit ?? MAX_UNBOUNDED_RECORD_BYTES;
+    for (const source of sources) {
+      for await (const line of linesFrom(source.path, source.gzip, maximumLineBytes)) {
+        const lineBytes = Buffer.byteLength(line) + 1;
+        if (bytes + lineBytes > query.max_bytes || examined >= query.max_records) {
+          truncated = true;
+          break;
+        }
+        bytes += lineBytes;
+        examined += 1;
+        if (!line.trim()) continue;
+        const record = parseLogRecord(line);
+        if (matches(record, query)) records.push(record);
+      }
+      if (truncated) break;
+    }
+    if (truncated) break;
+  }
+  return {
+    records,
+    records_examined: examined,
+    bytes_examined: bytes,
+    truncated,
+    expired_through: expiredThrough,
+  };
+}
+
+export function rotationFollowCursor(
+  family: HarneryRegisteredStorageFamily,
+): HarneryLogFollowCursor {
+  const directory = familyLogDirectory(family);
+  const manifest = readSegmentManifest(directory, family);
+  const active = join(directory, "active.jsonl");
+  return {
+    family_id: family.id,
+    manifest_sequence: manifest.next_sequence - 1,
+    active_offset: existsSync(active) ? regularFileStat(active).size : 0,
+  };
+}
+
+export async function readLogFollow(
+  family: HarneryRegisteredStorageFamily,
+  cursor: HarneryLogFollowCursor,
+  maxBytes: number,
+): Promise<{
+  records: readonly HarneryLogRecordV1[];
+  cursor: HarneryLogFollowCursor;
+  rotated: boolean;
+  history_expired: boolean;
+}> {
+  if (cursor.family_id !== family.id) throw new Error("follow cursor family mismatch");
+  if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0) {
+    throw new Error("follow requires a positive byte budget");
+  }
+  const directory = familyLogDirectory(family);
+  const manifest = readSegmentManifest(directory, family);
+  const rotated = manifest.next_sequence - 1 > cursor.manifest_sequence;
+  const prunedThrough = manifest.pruned_through_sequence ?? 0;
+  const historyExpired = prunedThrough > 0 && cursor.manifest_sequence <= prunedThrough;
+  const active = join(directory, "active.jsonl");
+  if (!existsSync(active)) {
+    return {
+      records: [],
+      cursor: rotationFollowCursor(family),
+      rotated,
+      history_expired: historyExpired,
+    };
+  }
+  const opened = openRegularNoFollow(active);
+  const maximumLineBytes =
+    family.policy.records.max_record_bytes.limit ?? MAX_UNBOUNDED_RECORD_BYTES;
+  const records: HarneryLogRecordV1[] = [];
+  let offset: number;
+  let consumed = 0;
+  try {
+    const size = fstatSync(opened.fd).size;
+    offset = rotated || cursor.active_offset > size ? 0 : cursor.active_offset;
+    let position = offset;
+    let remainingBudget = Math.min(maxBytes, size - offset);
+    let pending: Buffer[] = [];
+    let pendingBytes = 0;
+    while (remainingBudget > 0) {
+      const chunk = Buffer.allocUnsafe(Math.min(STREAM_CHUNK_BYTES, remainingBudget));
+      const bytesRead = readSync(opened.fd, chunk, 0, chunk.byteLength, position);
+      if (bytesRead === 0) break;
+      const view = chunk.subarray(0, bytesRead);
+      let start = 0;
+      for (let index = 0; index < view.byteLength; index += 1) {
+        if (view[index] !== 10) continue;
+        const part = view.subarray(start, index);
+        assertLineBound(active, pendingBytes + part.byteLength + 1, maximumLineBytes);
+        if (part.byteLength > 0) pending.push(part);
+        const line = Buffer.concat(pending, pendingBytes + part.byteLength).toString("utf8");
+        if (line) records.push(parseLogRecord(line));
+        consumed = position + index + 1 - offset;
+        pending = [];
+        pendingBytes = 0;
+        start = index + 1;
+      }
+      const tail = view.subarray(start);
+      assertLineBound(active, pendingBytes + tail.byteLength, maximumLineBytes);
+      if (tail.byteLength > 0) {
+        pending.push(tail);
+        pendingBytes += tail.byteLength;
+      }
+      position += bytesRead;
+      remainingBudget -= bytesRead;
+    }
+    assertOpenFileIdentity(active, opened.fd, opened.parent);
+  } finally {
+    closeSync(opened.fd);
+  }
+  return {
+    records,
+    cursor: {
+      family_id: family.id,
+      manifest_sequence: manifest.next_sequence - 1,
+      active_offset: offset + consumed,
+    },
+    rotated,
+    history_expired: historyExpired,
+  };
+}
+
+function manifestSources(
+  directory: string,
+  manifest: ReturnType<typeof readSegmentManifest>,
+): Array<{ path: string; gzip: boolean }> {
+  const sealed = manifest.segments.map((segment) => ({
+    path: join(directory, segment.file),
+    gzip: true,
+  }));
+  const active = join(directory, "active.jsonl");
+  return existsSync(active) ? [...sealed, { path: active, gzip: false }] : sealed;
+}
+
+async function* linesFrom(
+  path: string,
+  gzip: boolean,
+  maximumLineBytes: number,
+): AsyncGenerator<string> {
+  const opened = openRegularNoFollow(path);
+  const input = Readable.from(descriptorChunks(opened.fd));
+  const stream = gzip
+    ? input.pipe(
+        createGunzip({
+          chunkSize: Math.max(64, Math.min(STREAM_CHUNK_BYTES, maximumLineBytes + 1)),
+        }),
+      )
+    : input;
+  let pending: Buffer[] = [];
+  let pendingBytes = 0;
+  try {
+    for await (const value of stream) {
+      const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value as Uint8Array);
+      let start = 0;
+      for (let index = 0; index < chunk.byteLength; index += 1) {
+        if (chunk[index] !== 10) continue;
+        const part = chunk.subarray(start, index);
+        assertLineBound(path, pendingBytes + part.byteLength + 1, maximumLineBytes);
+        if (part.byteLength > 0) pending.push(part);
+        yield Buffer.concat(pending, pendingBytes + part.byteLength).toString("utf8");
+        pending = [];
+        pendingBytes = 0;
+        start = index + 1;
+      }
+      const tail = chunk.subarray(start);
+      assertLineBound(path, pendingBytes + tail.byteLength, maximumLineBytes);
+      if (tail.byteLength > 0) {
+        pending.push(tail);
+        pendingBytes += tail.byteLength;
+      }
+    }
+    if (pendingBytes > 0) throw new Error(`log query has a partial JSONL record: ${path}`);
+  } finally {
+    assertOpenFileIdentity(path, opened.fd, opened.parent);
+    stream.destroy();
+    input.destroy();
+    closeOwnedDescriptor(opened.fd);
+  }
+}
+
+function matches(record: HarneryLogRecordV1, query: HarneryLogQuery): boolean {
+  if (query.minimum_level && LEVELS.indexOf(record.level) < LEVELS.indexOf(query.minimum_level))
+    return false;
+  if (query.event && record.event !== query.event) return false;
+  if (query.since && record.emitted_at < query.since) return false;
+  if (query.until && record.emitted_at > query.until) return false;
+  return Object.entries(query.context ?? {}).every(([key, value]) => record.context[key] === value);
+}
+
+interface DirectoryIdentity {
+  path: string;
+  real_path: string;
+  dev: number;
+  ino: number;
+}
+
+function captureDirectoryIdentity(path: string): DirectoryIdentity {
+  const stat = lstatSync(path);
+  if (stat.isSymbolicLink()) throw new Error(`log query rejects symlink: ${path}`);
+  if (!stat.isDirectory()) throw new Error(`log query path is not a directory: ${path}`);
+  return { path, real_path: realpathSync(path), dev: stat.dev, ino: stat.ino };
+}
+
+function assertDirectoryIdentity(identity: DirectoryIdentity): void {
+  const current = lstatSync(identity.path);
+  if (
+    current.isSymbolicLink() ||
+    !current.isDirectory() ||
+    current.dev !== identity.dev ||
+    current.ino !== identity.ino ||
+    realpathSync(identity.path) !== identity.real_path
+  ) {
+    throw new Error(`log query directory identity changed: ${identity.path}`);
+  }
+}
+
+function openRegularNoFollow(path: string): { fd: number; parent: DirectoryIdentity } {
+  const parent = captureDirectoryIdentity(dirname(path));
+  rejectSymlink(path);
+  const fd = openSync(path, fsConstants.O_RDONLY | noFollowFlag());
+  try {
+    assertOpenFileIdentity(path, fd, parent);
+    return { fd, parent };
+  } catch (error) {
+    closeSync(fd);
+    throw error;
+  }
+}
+
+function assertOpenFileIdentity(path: string, fd: number, parent: DirectoryIdentity): void {
+  assertDirectoryIdentity(parent);
+  const opened = fstatSync(fd);
+  const current = lstatSync(path);
+  if (
+    !opened.isFile() ||
+    current.isSymbolicLink() ||
+    !current.isFile() ||
+    opened.dev !== current.dev ||
+    opened.ino !== current.ino
+  ) {
+    throw new Error(`log query path identity changed: ${path}`);
+  }
+  const fromParent = relative(parent.real_path, realpathSync(path));
+  if (fromParent.startsWith("..") || isAbsolute(fromParent)) {
+    throw new Error(`log query path escapes parent boundary: ${path}`);
+  }
+}
+
+function regularFileStat(path: string) {
+  const opened = openRegularNoFollow(path);
+  try {
+    return fstatSync(opened.fd);
+  } finally {
+    closeSync(opened.fd);
+  }
+}
+
+function rejectSymlink(path: string): void {
+  if (lstatSync(path).isSymbolicLink()) throw new Error(`log query rejects symlink: ${path}`);
+}
+
+function assertLineBound(path: string, bytes: number, maximum: number): void {
+  if (bytes > maximum) throw new Error(`log query record exceeds ${maximum} bytes: ${path}`);
+}
+
+function noFollowFlag(): number {
+  return fsConstants.O_NOFOLLOW ?? 0;
+}
+
+function closeOwnedDescriptor(fd: number): void {
+  try {
+    closeSync(fd);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== "EBADF") throw error;
+  }
+}
+
+async function* descriptorChunks(fd: number): AsyncGenerator<Buffer> {
+  let position = 0;
+  while (true) {
+    const chunk = Buffer.allocUnsafe(STREAM_CHUNK_BYTES);
+    const bytesRead = readSync(fd, chunk, 0, chunk.byteLength, position);
+    if (bytesRead === 0) return;
+    position += bytesRead;
+    yield chunk.subarray(0, bytesRead);
+  }
+}

package/src/core/storage/receipts.ts

@@ -0,0 +1,162 @@
+import { createHash, randomUUID } from "node:crypto";
+import {
+  closeSync,
+  existsSync,
+  fsyncSync,
+  lstatSync,
+  mkdirSync,
+  openSync,
+  readdirSync,
+  readFileSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname, join, relative, resolve, sep } from "node:path";
+import {
+  HARNERY_MAINTENANCE_RECEIPT_SCHEMA,
+  HarneryMaintenanceError,
+  type HarneryMaintenanceReceipt,
+} from "./maintenance.ts";
+
+export const HARNERY_RECEIPT_SEGMENT_MANIFEST_SCHEMA =
+  "harnery.storage-maintenance-receipt-segment/v1" as const;
+
+export interface HarneryReceiptConsolidationPlan {
+  schema: typeof HARNERY_RECEIPT_SEGMENT_MANIFEST_SCHEMA;
+  segment_id: string;
+  created_at: string;
+  source_count: number;
+  source_bytes: number;
+  source_paths_sha256: string;
+  payload_sha256: string;
+  payload_bytes: number;
+  applied: boolean;
+}
+
+/**
+ * Losslessly packs immutable maintenance receipts. The default is a pure plan;
+ * source replacement requires both an exact consolidation id and `yes`.
+ */
+export function consolidateMutationReceipts(
+  coordRoot: string,
+  options: {
+    now?: Date;
+    threshold?: number;
+    yes?: boolean;
+    consolidation_id?: string;
+  } = {},
+): HarneryReceiptConsolidationPlan | undefined {
+  const now = options.now ?? new Date();
+  if (!Number.isFinite(now.getTime()))
+    throw new HarneryMaintenanceError("invalid_time", "receipt consolidation");
+  const threshold = options.threshold ?? 1_000;
+  if (!Number.isSafeInteger(threshold) || threshold <= 1) {
+    throw new HarneryMaintenanceError("invalid_threshold", "receipt consolidation threshold");
+  }
+  const root = join(resolve(coordRoot), ".harnery", "maintenance", "receipts");
+  if (!existsSync(root)) return undefined;
+  const files = receiptFiles(root);
+  if (files.length < threshold) return undefined;
+  const rows = files.map((path) => readReceipt(path));
+  const payload = `${rows.map(({ receipt }) => JSON.stringify(receipt)).join("\n")}\n`;
+  const relativePaths = rows.map(({ path }) => relative(root, path).split(sep).join("/")).sort();
+  const segmentId =
+    options.consolidation_id ?? `receipts-${now.toISOString().slice(0, 10)}-${randomUUID()}`;
+  if (!/^[a-z0-9][a-z0-9._-]{0,127}$/.test(segmentId)) {
+    throw new HarneryMaintenanceError("invalid_id", "receipt consolidation id");
+  }
+  const plan: HarneryReceiptConsolidationPlan = {
+    schema: HARNERY_RECEIPT_SEGMENT_MANIFEST_SCHEMA,
+    segment_id: segmentId,
+    created_at: now.toISOString(),
+    source_count: rows.length,
+    source_bytes: rows.reduce((sum, row) => sum + row.bytes, 0),
+    source_paths_sha256: sha256(`${relativePaths.join("\n")}\n`),
+    payload_sha256: sha256(payload),
+    payload_bytes: Buffer.byteLength(payload),
+    applied: false,
+  };
+  if (!options.yes) return plan;
+  if (options.consolidation_id !== segmentId) {
+    throw new HarneryMaintenanceError(
+      "confirmation_required",
+      "receipt replacement requires an exact consolidation id and --yes",
+    );
+  }
+  const segments = join(root, "segments");
+  mkdirSync(segments, { recursive: true, mode: 0o700 });
+  const payloadPath = join(segments, `${segmentId}.jsonl`);
+  const manifestPath = join(segments, `${segmentId}.manifest.json`);
+  writeExclusiveDurable(payloadPath, payload);
+  const observed = readFileSync(payloadPath);
+  if (observed.byteLength !== plan.payload_bytes || sha256(observed) !== plan.payload_sha256) {
+    throw new HarneryMaintenanceError("receipt_segment_mismatch", segmentId);
+  }
+  const committed = { ...plan, applied: true };
+  writeExclusiveDurable(manifestPath, `${JSON.stringify(committed, null, 2)}\n`);
+  for (const row of rows) {
+    const current = readReceipt(row.path);
+    if (current.digest !== row.digest) {
+      throw new HarneryMaintenanceError("receipt_changed", relative(root, row.path));
+    }
+  }
+  for (const row of rows) unlinkSync(row.path);
+  return committed;
+}
+
+function receiptFiles(root: string): string[] {
+  const result: string[] = [];
+  for (const transaction of readdirSync(root, { withFileTypes: true })) {
+    if (transaction.name === "segments") continue;
+    const transactionPath = join(root, transaction.name);
+    const transactionStat = lstatSync(transactionPath);
+    if (!transaction.isDirectory() || transactionStat.isSymbolicLink()) continue;
+    for (const file of readdirSync(transactionPath, { withFileTypes: true })) {
+      if (!file.name.endsWith(".json")) continue;
+      result.push(join(transactionPath, file.name));
+    }
+  }
+  return result.sort();
+}
+
+function readReceipt(path: string): {
+  path: string;
+  bytes: number;
+  digest: string;
+  receipt: HarneryMaintenanceReceipt;
+} {
+  const stat = lstatSync(path);
+  if (!stat.isFile() || stat.isSymbolicLink() || stat.size <= 0 || stat.size > 256 * 1_024) {
+    throw new HarneryMaintenanceError("unsafe_receipt", path);
+  }
+  const bytes = readFileSync(path);
+  const receipt = JSON.parse(bytes.toString("utf8")) as HarneryMaintenanceReceipt;
+  if (
+    receipt.schema !== HARNERY_MAINTENANCE_RECEIPT_SCHEMA ||
+    typeof receipt.transaction_id !== "string" ||
+    typeof receipt.action_id !== "string" ||
+    !/^[a-f0-9]{64}$/.test(receipt.action_sha256)
+  ) {
+    throw new HarneryMaintenanceError("invalid_receipt", path);
+  }
+  return { path, bytes: bytes.byteLength, digest: sha256(bytes), receipt };
+}
+
+function writeExclusiveDurable(path: string, bytes: string): void {
+  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+  const temporary = `${path}.tmp-${process.pid}-${randomUUID()}`;
+  const fd = openSync(temporary, "wx", 0o600);
+  try {
+    writeFileSync(fd, bytes, "utf8");
+    fsyncSync(fd);
+  } finally {
+    closeSync(fd);
+  }
+  if (existsSync(path)) throw new HarneryMaintenanceError("segment_exists", path);
+  renameSync(temporary, path);
+}
+
+function sha256(value: string | Uint8Array): string {
+  return createHash("sha256").update(value).digest("hex");
+}