@tablestore/tablestore-agent-cli 0.3.1 → 0.4.1

package/dist/dashboard-runtime/dashboard/src/lib/file-memory-core.ts

@@ -0,0 +1,744 @@
+import type { FileMemoryItem, FileMemoryScope } from "@/types/file-memory";
+import { FatalRetrievalSecurityError } from "@/lib/retrieval-security";
+
+export const MAX_FILE_MEMORY_FILES = 300;
+export const MAX_FILE_MEMORY_BYTES = 3 * 1024 * 1024;
+export const MAX_FILE_MEMORY_SNAPSHOTS = 8;
+export const FILE_MEMORY_LIST_PAGE_SIZE = 100;
+export const FILE_MEMORY_READ_CONCURRENCY = 8;
+export const MAX_FILE_MEMORY_LIST_PAGES = 1_000;
+export const MAX_FILE_MEMORY_TOP_K = 50;
+export const MAX_FILE_MEMORY_SNIPPET_LINES = 10;
+export const MAX_FILE_MEMORY_SNIPPET_CHARS = 4_000;
+export const MAX_FILE_MEMORY_PATH_BYTES = 1_024;
+export const MAX_FILE_MEMORY_SHA_BYTES = 256;
+
+const utf8Encoder = new TextEncoder();
+
+export interface FileMemoryEntry {
+  rel: string;
+  size: number;
+  sha256: string;
+}
+
+export interface FileMemoryHit {
+  text: string;
+  source: string;
+  score: number;
+}
+
+export interface FileMemoryCacheSummary {
+  ok: boolean;
+  listed: number;
+  loaded: number;
+  reused: number;
+  totalBytes: number;
+  truncated: boolean;
+  error?: string;
+}
+
+interface CachedFile extends FileMemoryEntry {
+  path: string;
+  itemId: string;
+  text: string;
+}
+
+interface Snapshot {
+  files: Map<string, CachedFile>;
+}
+
+interface InflightLoad {
+  promise: Promise<LoadFileMemoryResult>;
+  controller: AbortController;
+  waiters: number;
+  settled: boolean;
+}
+
+export interface FileMemoryTransport {
+  listItems(
+    request: {
+      memoryStoreName: string;
+      scope: FileMemoryScope;
+      view: "basic";
+      limit: number;
+      nextToken?: string;
+    },
+    signal?: AbortSignal,
+  ): Promise<{ items: unknown[]; nextToken?: string }>;
+  getItem(
+    request: {
+      memoryStoreName: string;
+      scope: FileMemoryScope;
+      path: string;
+      view: "full";
+    },
+    signal?: AbortSignal,
+  ): Promise<{
+    content?: unknown;
+    itemId?: unknown;
+    scope?: unknown;
+    path?: unknown;
+    contentSha256?: unknown;
+    contentSizeBytes?: unknown;
+  }>;
+}
+
+export interface LoadFileMemoryOptions {
+  key: string;
+  memoryStoreName: string;
+  scope: FileMemoryScope;
+  transport: FileMemoryTransport;
+  signal?: AbortSignal;
+}
+
+export interface LoadFileMemoryResult {
+  key: string;
+  files: FileMemoryEntry[];
+  cache: FileMemoryCacheSummary;
+}
+
+function required(value: unknown, field: string): string {
+  if (typeof value !== "string" || !value.trim()) {
+    throw new Error(`file memory requires ${field}`);
+  }
+  return value.trim();
+}
+
+export function normalizeFileMemoryStoreName(memoryStoreName: string): string {
+  return required(memoryStoreName, "memoryStoreName");
+}
+
+function canonicalMetadataString(value: unknown, field: string): string {
+  const canonical = required(value, field);
+  if (value !== canonical) {
+    throw new Error(`file memory requires canonical ${field}`);
+  }
+  return canonical;
+}
+
+function canonicalFileMemorySha(value: unknown): string {
+  const sha = canonicalMetadataString(value, "contentSha256");
+  if (utf8Encoder.encode(sha).byteLength > MAX_FILE_MEMORY_SHA_BYTES) {
+    throw new Error("file memory contentSha256 exceeds the UTF-8 byte limit");
+  }
+  return sha;
+}
+
+export function normalizeFileMemoryScope(
+  scope: FileMemoryScope,
+): FileMemoryScope {
+  const read = (value: string, field: string) => {
+    const normalized = required(value, `an exact ${field}`);
+    if (normalized === "*") {
+      throw new Error(`file memory requires an exact ${field}`);
+    }
+    return normalized;
+  };
+  return {
+    appId: read(scope.appId, "appId"),
+    tenantId: read(scope.tenantId, "tenantId"),
+    agentId: read(scope.agentId, "agentId"),
+    runId: read(scope.runId, "runId"),
+  };
+}
+
+export function fileMemoryCacheKey(
+  namespace: string,
+  memoryStoreName: string,
+  scope: FileMemoryScope,
+): string {
+  const exact = normalizeFileMemoryScope(scope);
+  return JSON.stringify([
+    namespace,
+    normalizeFileMemoryStoreName(memoryStoreName),
+    exact.appId,
+    exact.tenantId,
+    exact.agentId,
+    exact.runId,
+  ]);
+}
+
+export function normalizeFileMemoryPath(path: string): {
+  path: string;
+  rel: string;
+} {
+  if (typeof path !== "string" || path.length === 0) {
+    throw new Error("file memory requires path");
+  }
+  const absolute = `/${path.normalize("NFC").replace(/^\/+/, "")}`;
+  const parts = absolute.slice(1).split("/");
+  if (
+    absolute === "/" ||
+    absolute.includes("\0") ||
+    absolute.includes("\\") ||
+    parts.some((part) => !part || part === "." || part === "..") ||
+    utf8Encoder.encode(absolute).byteLength > MAX_FILE_MEMORY_PATH_BYTES
+  ) {
+    throw new Error(`invalid file-memory path: ${path}`);
+  }
+  return { path: absolute, rel: parts.join("/") };
+}
+
+function sameScope(left: unknown, right: FileMemoryScope): boolean {
+  if (!left || typeof left !== "object") return false;
+  const value = left as Record<string, unknown>;
+  return (
+    value.appId === right.appId &&
+    value.tenantId === right.tenantId &&
+    value.agentId === right.agentId &&
+    value.runId === right.runId
+  );
+}
+
+function validateMetadata(
+  raw: unknown,
+  scope: FileMemoryScope,
+): FileMemoryItem {
+  if (!raw || typeof raw !== "object") {
+    throw new Error("invalid file-memory metadata");
+  }
+  const file = raw as FileMemoryItem;
+  normalizeFileMemoryPath(file.path);
+  const itemId = canonicalMetadataString(file.itemId, "itemId");
+  const latestVersionId = canonicalMetadataString(
+    file.latestVersionId,
+    "latestVersionId",
+  );
+  const sha = canonicalFileMemorySha(file.contentSha256);
+  if (
+    file.type !== "memoryfile" ||
+    !file.path.startsWith("/") ||
+    !Number.isSafeInteger(file.contentSizeBytes) ||
+    file.contentSizeBytes < 0 ||
+    !sameScope(file.scope, scope)
+  ) {
+    throw new Error(`invalid file-memory metadata: ${String(file.path)}`);
+  }
+  return {
+    ...file,
+    itemId,
+    latestVersionId,
+    contentSha256: sha,
+  };
+}
+
+function validateGetItemResponse(
+  response: Awaited<ReturnType<FileMemoryTransport["getItem"]>>,
+  metadata: FileMemoryItem,
+  scope: FileMemoryScope,
+): { text: string; size: number } {
+  if (typeof response.content !== "string") {
+    throw new Error(`GetItem omitted content for ${metadata.path}`);
+  }
+  const responsePath =
+    typeof response.path === "string"
+      ? normalizeFileMemoryPath(response.path).path
+      : undefined;
+  const metadataPath = normalizeFileMemoryPath(metadata.path).path;
+  const responseItemId = canonicalMetadataString(response.itemId, "itemId");
+  const responseSha = canonicalFileMemorySha(response.contentSha256);
+  if (
+    responseItemId !== metadata.itemId ||
+    !sameScope(response.scope, scope) ||
+    responsePath !== metadataPath ||
+    responseSha !== metadata.contentSha256
+  ) {
+    throw new Error(`GetItem identity mismatch for ${metadata.path}`);
+  }
+  return {
+    text: response.content,
+    size: utf8Encoder.encode(response.content).byteLength,
+  };
+}
+
+async function collectMetadata(
+  opts: LoadFileMemoryOptions,
+  scope: FileMemoryScope,
+): Promise<{ files: FileMemoryItem[]; truncated: boolean }> {
+  const files = new Map<string, FileMemoryItem>();
+  const tokens = new Set<string>();
+  let nextToken: string | undefined;
+
+  for (let page = 0; page < MAX_FILE_MEMORY_LIST_PAGES; page += 1) {
+    opts.signal?.throwIfAborted();
+    const response = await opts.transport.listItems(
+      {
+        memoryStoreName: opts.memoryStoreName,
+        scope,
+        view: "basic",
+        limit: FILE_MEMORY_LIST_PAGE_SIZE,
+        ...(nextToken ? { nextToken } : {}),
+      },
+      opts.signal,
+    );
+    opts.signal?.throwIfAborted();
+    if (!Array.isArray(response.items)) {
+      throw new Error("ListItems omitted items");
+    }
+    const itemEntries = response.items.filter(
+      (entry) =>
+        entry !== null &&
+        typeof entry === "object" &&
+        (entry as { type?: unknown }).type === "memoryfile",
+    );
+    for (const [index, raw] of itemEntries.entries()) {
+      const file = validateMetadata(raw, scope);
+      const { path } = normalizeFileMemoryPath(file.path);
+      if (!files.has(path)) files.set(path, { ...file, path });
+      if (files.size >= MAX_FILE_MEMORY_FILES) {
+        return {
+          files: [...files.values()],
+          truncated: index < itemEntries.length - 1 || !!response.nextToken,
+        };
+      }
+    }
+    if (!response.nextToken) {
+      return { files: [...files.values()], truncated: false };
+    }
+    if (tokens.has(response.nextToken)) {
+      throw new Error("ListItems repeated a pagination token");
+    }
+    tokens.add(response.nextToken);
+    nextToken = response.nextToken;
+  }
+  throw new Error("ListItems exceeded the pagination limit");
+}
+
+function entries(snapshot: Snapshot | undefined): FileMemoryEntry[] {
+  if (!snapshot) return [];
+  return [...snapshot.files.values()]
+    .map(({ rel, size, sha256 }) => ({ rel, size, sha256 }))
+    .sort((a, b) => a.rel.localeCompare(b.rel));
+}
+
+function tokenize(query: string): string[] {
+  const terms = new Set<string>();
+  for (const token of query.toLowerCase().match(/[a-z0-9]+/g) ?? []) {
+    if (token.length >= 2) terms.add(token);
+  }
+  for (const run of query.match(/[一-鿿㐀-䶿]+/g) ?? []) {
+    for (const char of run) terms.add(char);
+    for (let index = 0; index + 1 < run.length; index += 1) {
+      terms.add(run[index] + run[index + 1]);
+    }
+  }
+  return [...terms];
+}
+
+function score(text: string, terms: string[], query: string): number {
+  const lower = text.toLowerCase();
+  const matched = terms.reduce(
+    (count, term) => count + (lower.includes(term) ? 1 : 0),
+    0,
+  );
+  if (!matched) return 0;
+  const bonus =
+    query.length > 4 && lower.includes(query.toLowerCase()) ? 0.1 : 0;
+  return Math.min(1, matched / Math.max(1, terms.length) + bonus);
+}
+
+export class FileMemoryCache {
+  private readonly snapshots = new Map<string, Snapshot>();
+  private readonly inflight = new Map<string, InflightLoad>();
+
+  private read(key: string): Snapshot | undefined {
+    const snapshot = this.snapshots.get(key);
+    if (snapshot) {
+      this.snapshots.delete(key);
+      this.snapshots.set(key, snapshot);
+    }
+    return snapshot;
+  }
+
+  private store(key: string, snapshot: Snapshot): void {
+    this.snapshots.delete(key);
+    this.snapshots.set(key, snapshot);
+    while (this.snapshots.size > MAX_FILE_MEMORY_SNAPSHOTS) {
+      const oldest = this.snapshots.keys().next().value;
+      if (oldest === undefined) break;
+      this.snapshots.delete(oldest);
+    }
+  }
+
+  private deleteInflight(key: string, entry: InflightLoad): void {
+    if (this.inflight.get(key) === entry) this.inflight.delete(key);
+  }
+
+  async load(opts: LoadFileMemoryOptions): Promise<LoadFileMemoryResult> {
+    opts.signal?.throwIfAborted();
+    const memoryStoreName = normalizeFileMemoryStoreName(opts.memoryStoreName);
+    let entry = this.inflight.get(opts.key);
+    if (entry?.controller.signal.aborted) {
+      this.deleteInflight(opts.key, entry);
+      entry = undefined;
+    }
+    if (!entry) {
+      const controller = new AbortController();
+      entry = {
+        promise: Promise.resolve(undefined as never),
+        controller,
+        waiters: 0,
+        settled: false,
+      };
+      const created = entry;
+      created.promise = this.performLoad({
+        ...opts,
+        memoryStoreName,
+        signal: controller.signal,
+      }).finally(() => {
+        created.settled = true;
+        this.deleteInflight(opts.key, created);
+      });
+      this.inflight.set(opts.key, created);
+    }
+    return this.waitForInflight(entry, opts.signal);
+  }
+
+  private waitForInflight(
+    entry: InflightLoad,
+    signal?: AbortSignal,
+  ): Promise<LoadFileMemoryResult> {
+    entry.waiters += 1;
+    return new Promise((resolve, reject) => {
+      let released = false;
+      const release = (aborted: boolean) => {
+        if (released) return;
+        released = true;
+        entry.waiters -= 1;
+        if (aborted && entry.waiters === 0 && !entry.settled) {
+          entry.controller.abort(signal?.reason);
+        }
+      };
+      const onAbort = () => {
+        signal?.removeEventListener("abort", onAbort);
+        release(true);
+        reject(
+          signal?.reason ??
+            new DOMException("The operation was aborted", "AbortError"),
+        );
+      };
+      if (signal?.aborted) {
+        onAbort();
+        return;
+      }
+      signal?.addEventListener("abort", onAbort, { once: true });
+      entry.promise.then(
+        (result) => {
+          signal?.removeEventListener("abort", onAbort);
+          release(false);
+          resolve(result);
+        },
+        (error) => {
+          signal?.removeEventListener("abort", onAbort);
+          release(false);
+          reject(error);
+        },
+      );
+    });
+  }
+
+  private async performLoad(
+    opts: LoadFileMemoryOptions,
+  ): Promise<LoadFileMemoryResult> {
+    const scope = normalizeFileMemoryScope(opts.scope);
+    const previous = this.read(opts.key);
+    try {
+      const collected = await collectMetadata(opts, scope);
+      const selected = collected.files;
+      let truncated = collected.truncated;
+      const candidate = new Map<string, CachedFile>();
+      let totalBytes = 0;
+      let loadedCount = 0;
+      let reusedCount = 0;
+      const readController = new AbortController();
+      let stopReason: "budget" | "external" | "failure" | "fatal" | undefined;
+      let firstReadError: unknown;
+      let fatalReadError: FatalRetrievalSecurityError | undefined;
+      let notifyFatalRead!: (error: FatalRetrievalSecurityError) => void;
+      const fatalReadObserved = new Promise<FatalRetrievalSecurityError>(
+        (resolve) => {
+          notifyFatalRead = resolve;
+        },
+      );
+      const currentStopReason = () => stopReason;
+      const forwardAbort = () => {
+        stopReason = "external";
+        readController.abort(opts.signal?.reason);
+      };
+      if (opts.signal?.aborted) forwardAbort();
+      else opts.signal?.addEventListener("abort", forwardAbort, { once: true });
+
+      type ReadOutcome =
+        | { kind: "loaded"; file: CachedFile }
+        | { kind: "skipped" }
+        | { kind: "error"; error: unknown };
+      type IndexedReadOutcome = { index: number; outcome: ReadOutcome };
+      const active = new Map<number, Promise<IndexedReadOutcome>>();
+      const settled = new Map<number, ReadOutcome>();
+      let nextClaim = 0;
+      let nextCommit = 0;
+
+      const throwFatalRead = (): never => {
+        if (fatalReadError) throw fatalReadError;
+        throw new Error("file memory fatal read error is unavailable");
+      };
+
+      const throwIfReadStopped = async (): Promise<void> => {
+        if (stopReason === "external" || opts.signal?.aborted) {
+          await Promise.all(active.values());
+          opts.signal?.throwIfAborted();
+          return;
+        }
+        if (stopReason === "fatal") throwFatalRead();
+        if (stopReason !== "failure") return;
+
+        const activeReads = Promise.all(active.values());
+        await Promise.race([activeReads, fatalReadObserved]);
+        if (currentStopReason() === "external" || opts.signal?.aborted) {
+          await activeReads;
+          opts.signal?.throwIfAborted();
+          return;
+        }
+        if (currentStopReason() === "fatal") throwFatalRead();
+        await activeReads;
+        throw firstReadError;
+      };
+
+      const reusedFile = (metadata: FileMemoryItem): CachedFile | undefined => {
+        const { rel } = normalizeFileMemoryPath(metadata.path);
+        const cached = previous?.files.get(rel);
+        return cached?.sha256 === metadata.contentSha256 &&
+          cached.itemId === metadata.itemId
+          ? { ...cached, path: metadata.path, itemId: metadata.itemId }
+          : undefined;
+      };
+
+      const startRead = (
+        metadata: FileMemoryItem,
+        index: number,
+      ): Promise<IndexedReadOutcome> =>
+        (async (): Promise<ReadOutcome> => {
+          try {
+            readController.signal.throwIfAborted();
+            const response = await opts.transport.getItem(
+              {
+                memoryStoreName: opts.memoryStoreName,
+                scope,
+                path: metadata.path,
+                view: "full",
+              },
+              readController.signal,
+            );
+            readController.signal.throwIfAborted();
+            const actual = validateGetItemResponse(response, metadata, scope);
+            if (actual.size > MAX_FILE_MEMORY_BYTES) {
+              return { kind: "skipped" };
+            }
+            const { path, rel } = normalizeFileMemoryPath(metadata.path);
+            return {
+              kind: "loaded",
+              file: {
+                path,
+                rel,
+                itemId: metadata.itemId,
+                size: actual.size,
+                sha256: metadata.contentSha256,
+                text: actual.text,
+              },
+            };
+          } catch (error) {
+            if (stopReason === "external" || opts.signal?.aborted) {
+              return { kind: "error", error };
+            }
+            if (error instanceof FatalRetrievalSecurityError) {
+              if (stopReason !== "fatal") {
+                stopReason = "fatal";
+                fatalReadError = error;
+                notifyFatalRead(error);
+              }
+              readController.abort(
+                new DOMException("File memory security failure", "AbortError"),
+              );
+            } else if (!stopReason) {
+              stopReason = "failure";
+              firstReadError = error;
+              readController.abort(
+                new DOMException("File memory Get failed", "AbortError"),
+              );
+            }
+            return { kind: "error", error };
+          }
+        })().then((outcome) => ({ index, outcome }));
+
+      const claimReads = () => {
+        while (
+          !stopReason &&
+          nextClaim < selected.length &&
+          active.size + settled.size < FILE_MEMORY_READ_CONCURRENCY
+        ) {
+          const index = nextClaim;
+          nextClaim += 1;
+          const metadata = selected[index];
+          if (reusedFile(metadata)) continue;
+          active.set(index, startRead(metadata, index));
+        }
+      };
+
+      const commit = (file: CachedFile, source: "loaded" | "reused") => {
+        if (totalBytes + file.size > MAX_FILE_MEMORY_BYTES) {
+          truncated = true;
+          return;
+        }
+        candidate.set(file.rel, file);
+        totalBytes += file.size;
+        if (source === "loaded") loadedCount += 1;
+        else reusedCount += 1;
+      };
+
+      try {
+        claimReads();
+        while (nextCommit < selected.length) {
+          while (nextCommit < nextClaim) {
+            const metadata = selected[nextCommit];
+            const reused = reusedFile(metadata);
+            if (reused) {
+              commit(reused, "reused");
+              nextCommit += 1;
+            } else {
+              const outcome = settled.get(nextCommit);
+              if (!outcome) break;
+              settled.delete(nextCommit);
+              if (outcome.kind === "error") {
+                if (firstReadError === undefined) {
+                  firstReadError = outcome.error;
+                }
+              } else if (outcome.kind === "skipped") {
+                truncated = true;
+              } else {
+                commit(outcome.file, "loaded");
+              }
+              nextCommit += 1;
+            }
+
+            if (totalBytes === MAX_FILE_MEMORY_BYTES) {
+              if (nextCommit < selected.length) truncated = true;
+              if (!stopReason) {
+                stopReason = "budget";
+                readController.abort(
+                  new DOMException(
+                    "File memory byte budget reached",
+                    "AbortError",
+                  ),
+                );
+              }
+              break;
+            }
+          }
+
+          await throwIfReadStopped();
+          if (stopReason === "budget") {
+            await Promise.all(active.values());
+            break;
+          }
+
+          if (active.size === 0 && settled.size === 0) claimReads();
+          if (nextCommit >= selected.length) break;
+          if (active.size === 0) {
+            throw new Error("file memory ordered reader stalled");
+          }
+          const completed = await Promise.race(active.values());
+          active.delete(completed.index);
+          settled.set(completed.index, completed.outcome);
+          await throwIfReadStopped();
+        }
+      } finally {
+        opts.signal?.removeEventListener("abort", forwardAbort);
+        active.clear();
+        settled.clear();
+      }
+
+      const snapshot = { files: candidate };
+      this.store(opts.key, snapshot);
+      return {
+        key: opts.key,
+        files: entries(snapshot),
+        cache: {
+          ok: true,
+          listed: collected.files.length,
+          loaded: loadedCount,
+          reused: reusedCount,
+          totalBytes,
+          truncated,
+        },
+      };
+    } catch (error) {
+      if (opts.signal?.aborted) throw error;
+      this.snapshots.delete(opts.key);
+      if (error instanceof FatalRetrievalSecurityError) throw error;
+      return {
+        key: opts.key,
+        files: [],
+        cache: {
+          ok: false,
+          listed: 0,
+          loaded: 0,
+          reused: 0,
+          totalBytes: 0,
+          truncated: false,
+          error: error instanceof Error ? error.message : String(error),
+        },
+      };
+    }
+  }
+
+  readFile(key: string, rel: string): string {
+    const normalized = normalizeFileMemoryPath(rel).rel;
+    const file = this.read(key)?.files.get(normalized);
+    if (!file) throw new Error(`file-memory file not found: ${rel}`);
+    return file.text;
+  }
+
+  search(opts: { key: string; query: string; topK: number }): {
+    results: FileMemoryHit[];
+    skipped?: boolean;
+  } {
+    const query = opts.query.trim();
+    const snapshot = this.read(opts.key);
+    const terms = tokenize(query);
+    if (!query || !snapshot || terms.length === 0) {
+      return { results: [], skipped: true };
+    }
+    const hits: FileMemoryHit[] = [];
+    for (const file of snapshot.files.values()) {
+      const lines = file.text.split("\n");
+      for (
+        let index = 0;
+        index < lines.length;
+        index += MAX_FILE_MEMORY_SNIPPET_LINES
+      ) {
+        const text = lines
+          .slice(index, index + MAX_FILE_MEMORY_SNIPPET_LINES)
+          .join("\n")
+          .trim()
+          .slice(0, MAX_FILE_MEMORY_SNIPPET_CHARS);
+        if (!text) continue;
+        const hitScore = score(text, terms, query);
+        if (hitScore > 0) {
+          hits.push({ text, source: file.rel, score: hitScore });
+        }
+      }
+    }
+    hits.sort((a, b) => b.score - a.score || a.source.localeCompare(b.source));
+    const topK = Math.max(
+      1,
+      Math.min(MAX_FILE_MEMORY_TOP_K, Math.trunc(opts.topK) || 1),
+    );
+    return { results: hits.slice(0, topK) };
+  }
+
+  clear(): void {
+    this.snapshots.clear();
+    for (const entry of this.inflight.values()) entry.controller.abort();
+    this.inflight.clear();
+  }
+}