@tablestore/tablestore-agent-cli 0.2.10 → 0.3.1

package/dist/dashboard-runtime/dashboard/src/lib/browser-file-memory.ts

@@ -0,0 +1,397 @@
+"use client";
+
+import type { ChatMemoryWriteScope } from "@/components/chat/chat-config-context";
+import {
+  getItem as getItemService,
+  listItems as listItemsService,
+} from "@/services/file-memory";
+import type { FileMemoryItem, FileMemoryScope } from "@/types/file-memory";
+
+export const BROWSER_FILE_MEMORY_LABEL =
+  "标签页内容缓存 · 通过 Item API 按 SHA 复用";
+export const MAX_BROWSER_FILE_MEMORY_FILES = 300;
+export const MAX_BROWSER_FILE_MEMORY_BYTES = 3 * 1024 * 1024;
+export const MAX_BROWSER_FILE_MEMORY_SNAPSHOTS = 8;
+
+const LIST_PAGE_SIZE = 100;
+const READ_CONCURRENCY = 8;
+const MAX_LIST_PAGES = 1_000;
+const SNIPPET_LINES = 10;
+
+export interface BrowserFileMemoryEntry {
+  rel: string;
+  size: number;
+  sha256: string;
+}
+
+export interface BrowserFileMemoryHit {
+  text: string;
+  source: string;
+  score: number;
+}
+
+export interface BrowserFileMemoryCacheSummary {
+  ok: boolean;
+  listed: number;
+  loaded: number;
+  reused: number;
+  totalBytes: number;
+  truncated: boolean;
+  error?: string;
+}
+
+interface CachedFile extends BrowserFileMemoryEntry {
+  path: string;
+  itemId: string;
+  text: string;
+}
+
+interface BrowserFileMemorySnapshot {
+  files: Map<string, CachedFile>;
+  updatedAt: number;
+}
+
+type ListItems = typeof listItemsService;
+type GetItem = typeof getItemService;
+
+export interface LoadBrowserFileMemoryOptions {
+  dataScopeKey: string;
+  memoryStoreName: string;
+  scope: ChatMemoryWriteScope;
+  listItems?: ListItems;
+  getItem?: GetItem;
+}
+
+export interface LoadBrowserFileMemoryResult {
+  key: string;
+  files: BrowserFileMemoryEntry[];
+  cache: BrowserFileMemoryCacheSummary;
+}
+
+const snapshots = new Map<string, BrowserFileMemorySnapshot>();
+const inflight = new Map<string, Promise<LoadBrowserFileMemoryResult>>();
+
+function readSnapshot(key: string): BrowserFileMemorySnapshot | undefined {
+  const snapshot = snapshots.get(key);
+  if (snapshot) {
+    snapshots.delete(key);
+    snapshots.set(key, snapshot);
+  }
+  return snapshot;
+}
+
+function storeSnapshot(key: string, snapshot: BrowserFileMemorySnapshot): void {
+  snapshots.delete(key);
+  snapshots.set(key, snapshot);
+  while (snapshots.size > MAX_BROWSER_FILE_MEMORY_SNAPSHOTS) {
+    const oldestKey = snapshots.keys().next().value;
+    if (oldestKey === undefined) break;
+    snapshots.delete(oldestKey);
+  }
+}
+
+function normalizeScope(scope: ChatMemoryWriteScope): FileMemoryScope {
+  const read = (value: string, field: string) => {
+    const normalized = value.trim();
+    if (!normalized || normalized === "*") {
+      throw new Error(`file memory requires an exact ${field}`);
+    }
+    return normalized;
+  };
+  return {
+    appId: read(scope.appId, "appId"),
+    tenantId: read(scope.tenantId, "tenantId"),
+    agentId: read(scope.agentId, "agentId"),
+    runId: read(scope.runId, "runId"),
+  };
+}
+
+export function browserFileMemoryKey(opts: {
+  dataScopeKey: string;
+  memoryStoreName: string;
+  scope: ChatMemoryWriteScope;
+}): string {
+  const scope = normalizeScope(opts.scope);
+  return JSON.stringify([
+    opts.dataScopeKey,
+    opts.memoryStoreName,
+    scope.appId,
+    scope.tenantId,
+    scope.agentId,
+    scope.runId,
+  ]);
+}
+
+function normalizePath(path: string): { path: string; rel: string } {
+  const absolute = `/${path.trim().replace(/^\/+/, "")}`;
+  const parts = absolute.slice(1).split("/");
+  if (
+    absolute === "/" ||
+    absolute.includes("\\") ||
+    parts.some((part) => !part || part === "." || part === "..")
+  ) {
+    throw new Error(`invalid file-memory path: ${path}`);
+  }
+  return { path: absolute, rel: parts.join("/") };
+}
+
+function validateMetadata(file: FileMemoryItem): void {
+  normalizePath(file.path);
+  if (
+    file.type !== "memoryfile" ||
+    !file.itemId ||
+    !file.latestVersionId ||
+    !file.contentSha256 ||
+    !Number.isSafeInteger(file.contentSizeBytes) ||
+    file.contentSizeBytes < 0
+  ) {
+    throw new Error(`invalid file-memory metadata: ${file.path}`);
+  }
+}
+
+async function collectMetadata(
+  opts: LoadBrowserFileMemoryOptions,
+  scope: FileMemoryScope,
+): Promise<{ files: FileMemoryItem[]; truncated: boolean }> {
+  const list = opts.listItems ?? listItemsService;
+  const files = new Map<string, FileMemoryItem>();
+  const tokens = new Set<string>();
+  let nextToken: string | undefined;
+  let truncated = false;
+
+  for (let page = 0; page < MAX_LIST_PAGES; page += 1) {
+    const response = await list(opts.memoryStoreName, {
+      scope,
+      view: "basic",
+      limit: LIST_PAGE_SIZE,
+      ...(nextToken ? { nextToken } : {}),
+    });
+    const itemEntries = response.items.filter(
+      (entry): entry is FileMemoryItem => entry.type === "memoryfile",
+    );
+    for (const [index, file] of itemEntries.entries()) {
+      validateMetadata(file);
+      const { path } = normalizePath(file.path);
+      if (!files.has(path)) files.set(path, { ...file, path });
+      if (files.size >= MAX_BROWSER_FILE_MEMORY_FILES) {
+        truncated = index < itemEntries.length - 1 || !!response.nextToken;
+        return { files: [...files.values()], truncated };
+      }
+    }
+    if (!response.nextToken) return { files: [...files.values()], truncated };
+    if (tokens.has(response.nextToken)) {
+      throw new Error("ListItems repeated a pagination token");
+    }
+    tokens.add(response.nextToken);
+    nextToken = response.nextToken;
+  }
+  throw new Error("ListItems exceeded the pagination limit");
+}
+
+async function mapConcurrent<T, TResult>(
+  items: T[],
+  concurrency: number,
+  worker: (item: T) => Promise<TResult>,
+): Promise<TResult[]> {
+  const results = new Array<TResult>(items.length);
+  let cursor = 0;
+  await Promise.all(
+    Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+      while (cursor < items.length) {
+        const index = cursor;
+        cursor += 1;
+        results[index] = await worker(items[index]);
+      }
+    }),
+  );
+  return results;
+}
+
+function snapshotEntries(
+  snapshot: BrowserFileMemorySnapshot | undefined,
+): BrowserFileMemoryEntry[] {
+  if (!snapshot) return [];
+  return [...snapshot.files.values()]
+    .map(({ rel, size, sha256 }) => ({ rel, size, sha256 }))
+    .sort((a, b) => a.rel.localeCompare(b.rel));
+}
+
+async function performLoad(
+  opts: LoadBrowserFileMemoryOptions,
+  key: string,
+  scope: FileMemoryScope,
+): Promise<LoadBrowserFileMemoryResult> {
+  const previous = readSnapshot(key);
+  try {
+    const collected = await collectMetadata(opts, scope);
+    const selected: FileMemoryItem[] = [];
+    let totalBytes = 0;
+    let truncated = collected.truncated;
+    for (const file of collected.files) {
+      if (totalBytes + file.contentSizeBytes > MAX_BROWSER_FILE_MEMORY_BYTES) {
+        truncated = true;
+        continue;
+      }
+      selected.push(file);
+      totalBytes += file.contentSizeBytes;
+    }
+
+    const candidate = new Map<string, CachedFile>();
+    const toRead: FileMemoryItem[] = [];
+    let reused = 0;
+    for (const file of selected) {
+      const { rel } = normalizePath(file.path);
+      const cached = previous?.files.get(rel);
+      if (cached?.sha256 === file.contentSha256) {
+        candidate.set(rel, {
+          ...cached,
+          path: file.path,
+          itemId: file.itemId,
+          size: file.contentSizeBytes,
+        });
+        reused += 1;
+      } else {
+        toRead.push(file);
+      }
+    }
+
+    const read = opts.getItem ?? getItemService;
+    const loaded = await mapConcurrent(
+      toRead,
+      READ_CONCURRENCY,
+      async (file) => {
+        const response = await read(opts.memoryStoreName, {
+          scope,
+          path: file.path,
+          view: "full",
+        });
+        if (response.content === undefined) {
+          throw new Error(`GetItem omitted content for ${file.path}`);
+        }
+        const { path, rel } = normalizePath(file.path);
+        return {
+          path,
+          rel,
+          itemId: file.itemId,
+          size: file.contentSizeBytes,
+          sha256: file.contentSha256,
+          text: response.content,
+        } satisfies CachedFile;
+      },
+    );
+    for (const file of loaded) candidate.set(file.rel, file);
+
+    const snapshot = { files: candidate, updatedAt: Date.now() };
+    storeSnapshot(key, snapshot);
+    return {
+      key,
+      files: snapshotEntries(snapshot),
+      cache: {
+        ok: true,
+        listed: collected.files.length,
+        loaded: loaded.length,
+        reused,
+        totalBytes,
+        truncated,
+      },
+    };
+  } catch (error) {
+    const files = snapshotEntries(previous);
+    return {
+      key,
+      files,
+      cache: {
+        ok: false,
+        listed: files.length,
+        loaded: 0,
+        reused: files.length,
+        totalBytes: files.reduce((sum, file) => sum + file.size, 0),
+        truncated: false,
+        error: error instanceof Error ? error.message : String(error),
+      },
+    };
+  }
+}
+
+export function loadBrowserFileMemory(
+  opts: LoadBrowserFileMemoryOptions,
+): Promise<LoadBrowserFileMemoryResult> {
+  const scope = normalizeScope(opts.scope);
+  const key = browserFileMemoryKey({ ...opts, scope });
+  const existing = inflight.get(key);
+  if (existing) return existing;
+  const run = performLoad(opts, key, scope).finally(() => inflight.delete(key));
+  inflight.set(key, run);
+  return run;
+}
+
+export function readBrowserFileMemoryFile(key: string, rel: string): string {
+  const normalized = normalizePath(rel).rel;
+  const file = readSnapshot(key)?.files.get(normalized);
+  if (!file) throw new Error(`browser file-memory file not found: ${rel}`);
+  return file.text;
+}
+
+function tokenizeQuery(query: string): string[] {
+  const terms = new Set<string>();
+  for (const token of query.toLowerCase().match(/[a-z0-9]+/g) ?? []) {
+    if (token.length >= 2) terms.add(token);
+  }
+  for (const run of query.match(/[一-鿿㐀-䶿]+/g) ?? []) {
+    for (const char of run) terms.add(char);
+    for (let index = 0; index + 1 < run.length; index += 1) {
+      terms.add(run[index] + run[index + 1]);
+    }
+  }
+  return [...terms];
+}
+
+function scoreSnippet(
+  text: string,
+  terms: string[],
+  fullQuery: string,
+): number {
+  const lower = text.toLowerCase();
+  const matched = terms.reduce(
+    (count, term) => count + (lower.includes(term) ? 1 : 0),
+    0,
+  );
+  if (!matched) return 0;
+  const bonus =
+    fullQuery.length > 4 && lower.includes(fullQuery.toLowerCase()) ? 0.1 : 0;
+  return Math.min(1, matched / Math.max(1, terms.length) + bonus);
+}
+
+export function searchBrowserFileMemory(opts: {
+  key: string;
+  query: string;
+  topK: number;
+}): { results: BrowserFileMemoryHit[]; skipped?: boolean } {
+  const query = opts.query.trim();
+  const snapshot = readSnapshot(opts.key);
+  const terms = tokenizeQuery(query);
+  if (!query || !snapshot || terms.length === 0) {
+    return { results: [], skipped: true };
+  }
+  const hits: BrowserFileMemoryHit[] = [];
+  for (const file of snapshot.files.values()) {
+    const lines = file.text.split("\n");
+    for (let index = 0; index < lines.length; index += SNIPPET_LINES) {
+      const text = lines
+        .slice(index, index + SNIPPET_LINES)
+        .join("\n")
+        .trim();
+      if (!text) continue;
+      const score = scoreSnippet(text, terms, query);
+      if (score > 0) hits.push({ text, source: file.rel, score });
+    }
+  }
+  hits.sort((a, b) => b.score - a.score || a.source.localeCompare(b.source));
+  const topK = Math.max(1, Math.min(50, Math.trunc(opts.topK) || 1));
+  return { results: hits.slice(0, topK) };
+}
+
+export function clearBrowserFileMemoryForTests(): void {
+  snapshots.clear();
+  inflight.clear();
+}

package/dist/dashboard-runtime/dashboard/src/lib/file-scope-history.ts

@@ -0,0 +1,108 @@
+import type { FileMemoryScope } from "@/types/file-memory";
+
+export const MAX_RECENT_FILE_SCOPES = 10;
+const STORAGE_KEY_PREFIX = "tsagent.fileScopes.v1";
+
+type ScopeStorage = Pick<Storage, "getItem" | "setItem">;
+
+export function getRecentFileScopeStorageKey(
+  dataScopeKey: string,
+  memoryStoreName: string,
+): string {
+  return `${STORAGE_KEY_PREFIX}:${JSON.stringify([dataScopeKey, memoryStoreName])}`;
+}
+
+export function normalizeExactFileMemoryScope(
+  value: Partial<FileMemoryScope> | null | undefined,
+): FileMemoryScope | null {
+  if (!value) return null;
+  const fields = [value.appId, value.tenantId, value.agentId, value.runId];
+  if (
+    fields.some(
+      (field) =>
+        typeof field !== "string" || !field.trim() || field.trim() === "*",
+    )
+  ) {
+    return null;
+  }
+  return {
+    appId: value.appId!.trim(),
+    tenantId: value.tenantId!.trim(),
+    agentId: value.agentId!.trim(),
+    runId: value.runId!.trim(),
+  };
+}
+
+export function isExactFileMemoryScope(
+  value: Partial<FileMemoryScope> | null | undefined,
+): value is FileMemoryScope {
+  return normalizeExactFileMemoryScope(value) !== null;
+}
+
+export function applyFileScopeToSearchParams(
+  current: URLSearchParams,
+  scope: FileMemoryScope,
+): URLSearchParams {
+  const next = new URLSearchParams(current);
+  next.set("view", "files");
+  next.set("appId", scope.appId);
+  next.set("tenantId", scope.tenantId);
+  next.set("agentId", scope.agentId);
+  next.set("runId", scope.runId);
+  next.delete("file");
+  return next;
+}
+
+export function readRecentFileScopes(
+  storage: Pick<ScopeStorage, "getItem">,
+  dataScopeKey: string,
+  memoryStoreName: string,
+): FileMemoryScope[] {
+  try {
+    const raw = storage.getItem(
+      getRecentFileScopeStorageKey(dataScopeKey, memoryStoreName),
+    );
+    if (!raw) return [];
+    const parsed = JSON.parse(raw) as unknown;
+    if (!Array.isArray(parsed)) return [];
+    return parsed
+      .map((item) =>
+        normalizeExactFileMemoryScope(
+          item && typeof item === "object"
+            ? (item as Partial<FileMemoryScope>)
+            : null,
+        ),
+      )
+      .filter((item): item is FileMemoryScope => item !== null)
+      .slice(0, MAX_RECENT_FILE_SCOPES);
+  } catch {
+    return [];
+  }
+}
+
+export function rememberRecentFileScope(
+  storage: ScopeStorage,
+  dataScopeKey: string,
+  memoryStoreName: string,
+  value: Partial<FileMemoryScope>,
+): FileMemoryScope[] {
+  const scope = normalizeExactFileMemoryScope(value);
+  if (!scope)
+    return readRecentFileScopes(storage, dataScopeKey, memoryStoreName);
+  const identity = JSON.stringify(scope);
+  const next = [
+    scope,
+    ...readRecentFileScopes(storage, dataScopeKey, memoryStoreName).filter(
+      (item) => JSON.stringify(item) !== identity,
+    ),
+  ].slice(0, MAX_RECENT_FILE_SCOPES);
+  try {
+    storage.setItem(
+      getRecentFileScopeStorageKey(dataScopeKey, memoryStoreName),
+      JSON.stringify(next),
+    );
+  } catch {
+    // Recent navigation history must never block file-memory access.
+  }
+  return next;
+}

package/dist/dashboard-runtime/dashboard/src/lib/memory-scope-history.ts

@@ -0,0 +1,165 @@
+import type { MemoryScope } from "@/types/memory";
+
+export const MAX_RECENT_MEMORY_SCOPES = 10;
+const STORAGE_KEY_PREFIX = "tsagent.memoryScopes.v1";
+
+type ScopeStorage = Pick<Storage, "getItem" | "setItem" | "removeItem">;
+type OptionalScopeField = "tenantId" | "agentId" | "runId";
+
+const OPTIONAL_SCOPE_FIELDS: OptionalScopeField[] = [
+  "tenantId",
+  "agentId",
+  "runId",
+];
+
+export function getRecentMemoryScopeStorageKey(
+  dataScopeKey: string,
+  memoryStoreName: string,
+): string {
+  return `${STORAGE_KEY_PREFIX}:${JSON.stringify([dataScopeKey, memoryStoreName])}`;
+}
+
+export function isAllWildcardMemoryScope(
+  value: Partial<MemoryScope> | null | undefined,
+): boolean {
+  if (!value || typeof value.appId !== "string" || value.appId.trim() !== "*") {
+    return false;
+  }
+
+  return OPTIONAL_SCOPE_FIELDS.every((field) => {
+    const fieldValue = value[field];
+    if (fieldValue === undefined) return true;
+    if (typeof fieldValue !== "string") return false;
+    const normalized = fieldValue.trim();
+    return !normalized || normalized === "*";
+  });
+}
+
+export function normalizeRecentMemoryScope(
+  value: Partial<MemoryScope> | null | undefined,
+): MemoryScope | null {
+  if (!value || typeof value.appId !== "string" || !value.appId.trim()) {
+    return null;
+  }
+
+  const scope: MemoryScope = { appId: value.appId.trim() };
+  for (const field of OPTIONAL_SCOPE_FIELDS) {
+    const fieldValue = value[field];
+    if (typeof fieldValue === "string" && fieldValue.trim()) {
+      scope[field] = fieldValue.trim();
+    }
+  }
+
+  return isAllWildcardMemoryScope(scope) ? null : scope;
+}
+
+function scopeIdentity(scope: MemoryScope): string {
+  return JSON.stringify([
+    scope.appId,
+    scope.tenantId ?? "",
+    scope.agentId ?? "",
+    scope.runId ?? "",
+  ]);
+}
+
+export function readRecentMemoryScopes(
+  storage: Pick<ScopeStorage, "getItem">,
+  dataScopeKey: string,
+  memoryStoreName: string,
+): MemoryScope[] {
+  try {
+    const raw = storage.getItem(
+      getRecentMemoryScopeStorageKey(dataScopeKey, memoryStoreName),
+    );
+    if (!raw) return [];
+
+    const parsed = JSON.parse(raw) as unknown;
+    if (!Array.isArray(parsed)) return [];
+
+    const identities = new Set<string>();
+    const scopes: MemoryScope[] = [];
+    for (const item of parsed) {
+      const value =
+        item && typeof item === "object"
+          ? (item as Partial<MemoryScope>)
+          : null;
+      if (isAllWildcardMemoryScope(value)) continue;
+
+      const scope = normalizeRecentMemoryScope(value);
+      if (!scope) continue;
+
+      const identity = scopeIdentity(scope);
+      if (identities.has(identity)) continue;
+      identities.add(identity);
+      scopes.push(scope);
+      if (scopes.length === MAX_RECENT_MEMORY_SCOPES) break;
+    }
+    return scopes;
+  } catch {
+    return [];
+  }
+}
+
+export function rememberRecentMemoryScope(
+  storage: Pick<ScopeStorage, "getItem" | "setItem">,
+  dataScopeKey: string,
+  memoryStoreName: string,
+  value: Partial<MemoryScope>,
+): MemoryScope[] {
+  const current = readRecentMemoryScopes(
+    storage,
+    dataScopeKey,
+    memoryStoreName,
+  );
+  if (isAllWildcardMemoryScope(value)) return current;
+
+  const scope = normalizeRecentMemoryScope(value);
+  if (!scope) return current;
+
+  const identity = scopeIdentity(scope);
+  const next = [
+    scope,
+    ...current.filter((item) => scopeIdentity(item) !== identity),
+  ].slice(0, MAX_RECENT_MEMORY_SCOPES);
+
+  try {
+    storage.setItem(
+      getRecentMemoryScopeStorageKey(dataScopeKey, memoryStoreName),
+      JSON.stringify(next),
+    );
+  } catch {
+    // Scope history must never block memory navigation.
+  }
+  return next;
+}
+
+export function clearRecentMemoryScopes(
+  storage: Pick<ScopeStorage, "removeItem">,
+  dataScopeKey: string,
+  memoryStoreName: string,
+): void {
+  try {
+    storage.removeItem(
+      getRecentMemoryScopeStorageKey(dataScopeKey, memoryStoreName),
+    );
+  } catch {
+    // Scope history must never block memory navigation.
+  }
+}
+
+export function excludeRecentMemoryScopes(
+  knownScopes: readonly MemoryScope[],
+  recentScopes: readonly MemoryScope[],
+): MemoryScope[] {
+  const recentIdentities = new Set(
+    recentScopes
+      .map(normalizeRecentMemoryScope)
+      .filter((scope): scope is MemoryScope => scope !== null)
+      .map(scopeIdentity),
+  );
+
+  return knownScopes.filter((scope) => {
+    const normalized = normalizeRecentMemoryScope(scope);
+    return !normalized || !recentIdentities.has(scopeIdentity(normalized));
+  });
+}