@hassangameryt/freeflow 0.3.0 → 0.5.0

package/router/dist/local/local-traversal.d.ts

@@ -0,0 +1,18 @@
+export interface LocalTextFileRef {
+  root: string;
+  path: string;
+  absolutePath: string;
+  sizeBytes: number;
+}
+export interface ResolvedLocalPath {
+  root: string;
+  absolutePath: string;
+  relativePath: string;
+}
+export interface CollectLocalTextFileRefsOptions {
+  root: string;
+  requestedPath?: string;
+  generatedPathGlobs?: readonly string[];
+}
+export declare function collectLocalTextFileRefs(options: CollectLocalTextFileRefsOptions): Promise<LocalTextFileRef[]>;
+export declare function resolveLocalPath(root: string, requestedPath?: string): Promise<ResolvedLocalPath>;

package/router/dist/local/local-traversal.js

@@ -0,0 +1,334 @@
+import { realpath, readdir, stat } from "node:fs/promises";
+import { homedir, tmpdir } from "node:os";
+import { basename, isAbsolute, parse, relative, resolve } from "node:path";
+import { matchesGeneratedPathGlob } from "../repo/repo-traversal.js";
+const BROAD_SCAN_MAX_FILE_BYTES = 1024 * 1024;
+const LOCAL_SCAN_MAX_DIRECTORIES = 2_000;
+const LOCAL_SCAN_MAX_FILES = 10_000;
+const LOCAL_SCAN_MAX_TOTAL_BYTES = 64 * 1024 * 1024;
+const BROAD_SKIP_DIRS = new Set([
+  ".git",
+  "node_modules",
+  "dist",
+  "build",
+  "out",
+  ".next",
+  ".nuxt",
+  "coverage",
+  "target",
+  "graphify-out",
+  ".cache",
+  ".tmp",
+  "tmp",
+  "temp",
+  "logs",
+  "generated",
+]);
+const SECRET_DIRS = new Set([".aws", ".azure", ".docker", ".gnupg", ".kube", ".password-store", ".ssh"]);
+const BROAD_SKIP_FILE_EXTENSIONS = new Set([
+  ".7z",
+  ".avi",
+  ".bin",
+  ".bmp",
+  ".br",
+  ".class",
+  ".db",
+  ".dll",
+  ".dylib",
+  ".eot",
+  ".exe",
+  ".gif",
+  ".gz",
+  ".heic",
+  ".icns",
+  ".ico",
+  ".jar",
+  ".jpeg",
+  ".jpg",
+  ".mov",
+  ".mp3",
+  ".mp4",
+  ".otf",
+  ".parquet",
+  ".pdf",
+  ".png",
+  ".pyc",
+  ".rar",
+  ".so",
+  ".sqlite",
+  ".sqlite3",
+  ".tar",
+  ".tgz",
+  ".ttf",
+  ".wasm",
+  ".webm",
+  ".webp",
+  ".woff",
+  ".woff2",
+  ".zip",
+  ".zst",
+]);
+const SECRET_FILE_NAMES = new Set([
+  ".env",
+  ".netrc",
+  ".npmrc",
+  ".pypirc",
+  "authorized_keys",
+  "credentials",
+  "credentials.json",
+  "id_dsa",
+  "id_ecdsa",
+  "id_ed25519",
+  "id_rsa",
+  "known_hosts",
+]);
+export async function collectLocalTextFileRefs(options) {
+  const root = await resolveLocalRoot(options.root);
+  const start = await resolveLocalRequestedPath(root, options.requestedPath);
+  const fileRefs = [];
+  const visitedDirectories = new Set();
+  const broadTraversal = !isExplicitRequestedPath(start.relativePath);
+  const budget = { directories: 0, files: 0, bytes: 0 };
+  await collectTextFileRefs({
+    root,
+    currentPath: start.absolutePath,
+    fileRefs,
+    broadTraversal,
+    visitedDirectories,
+    generatedPathGlobs: options.generatedPathGlobs ?? [],
+    budget,
+  });
+  return fileRefs;
+}
+export async function resolveLocalPath(root, requestedPath) {
+  const rootRealPath = await resolveLocalRoot(root);
+  return resolveLocalRequestedPath(rootRealPath, requestedPath);
+}
+async function resolveLocalRoot(root) {
+  if (typeof root !== "string" || root.trim().length === 0) {
+    throw new Error("Local source.root must be a non-empty absolute path.");
+  }
+  if (!isAbsolute(root)) {
+    throw new Error(`Local source.root must be an absolute path: ${root}`);
+  }
+  const rootRealPath = await realpath(resolve(root));
+  const rootStat = await stat(rootRealPath);
+  if (!rootStat.isDirectory()) {
+    throw new Error(`Local source.root must be a directory: ${root}`);
+  }
+  const broadRootReason = broadRootRejectionReason(rootRealPath);
+  if (broadRootReason) {
+    throw new Error(
+      `Local source.root is too broad: ${rootRealPath}. ${broadRootReason} Choose a narrower explicit docs/source directory.`,
+    );
+  }
+  return rootRealPath;
+}
+async function resolveLocalRequestedPath(rootRealPath, requestedPath) {
+  if (requestedPath !== undefined && requestedPath !== "" && isAbsolute(requestedPath)) {
+    throw new Error(`Local source.path must be relative to source.root: ${requestedPath}`);
+  }
+  const requestedAbsolutePath =
+    requestedPath && requestedPath !== "" ? resolve(rootRealPath, requestedPath) : rootRealPath;
+  const requestedRealPath = await realpath(requestedAbsolutePath);
+  if (!isPathInsideRoot(rootRealPath, requestedRealPath)) {
+    throw new Error(`Local path escapes root: ${requestedPath ?? "."}`);
+  }
+  return {
+    root: rootRealPath,
+    absolutePath: requestedRealPath,
+    relativePath: normalizeRelativePath(relative(rootRealPath, requestedRealPath)),
+  };
+}
+async function collectTextFileRefs(options) {
+  let currentRealPath;
+  try {
+    currentRealPath = await realpath(options.currentPath);
+  } catch {
+    return;
+  }
+  if (!isPathInsideRoot(options.root, currentRealPath)) {
+    return;
+  }
+  let currentStat;
+  try {
+    currentStat = await stat(currentRealPath);
+  } catch {
+    return;
+  }
+  const path = normalizeRelativePath(relative(options.root, currentRealPath));
+  if (isSensitivePath(path)) {
+    return;
+  }
+  if (currentStat.isDirectory()) {
+    if (options.visitedDirectories.has(currentRealPath)) {
+      return;
+    }
+    options.visitedDirectories.add(currentRealPath);
+    noteDirectory(options.budget);
+    if (options.broadTraversal && path !== "" && shouldSkipBroadDirectory(path, options.generatedPathGlobs)) {
+      return;
+    }
+    const entries = await readdir(currentRealPath, { withFileTypes: true });
+    for (const entry of entries) {
+      await collectTextFileRefs({
+        ...options,
+        currentPath: resolve(currentRealPath, entry.name),
+      });
+    }
+    return;
+  }
+  if (!currentStat.isFile()) {
+    return;
+  }
+  noteFile(options.budget, currentStat.size);
+  if (options.broadTraversal && shouldSkipBroadFile(path, currentStat.size, options.generatedPathGlobs)) {
+    return;
+  }
+  options.fileRefs.push({ root: options.root, path, absolutePath: currentRealPath, sizeBytes: currentStat.size });
+}
+function noteDirectory(budget) {
+  budget.directories += 1;
+  if (budget.directories > LOCAL_SCAN_MAX_DIRECTORIES) {
+    throw new Error(
+      `Local broad scan exceeded directory budget (${LOCAL_SCAN_MAX_DIRECTORIES}); narrow source.root or source.path.`,
+    );
+  }
+}
+function noteFile(budget, size) {
+  budget.files += 1;
+  budget.bytes += size;
+  if (budget.files > LOCAL_SCAN_MAX_FILES) {
+    throw new Error(
+      `Local broad scan exceeded file budget (${LOCAL_SCAN_MAX_FILES}); narrow source.root or source.path.`,
+    );
+  }
+  if (budget.bytes > LOCAL_SCAN_MAX_TOTAL_BYTES) {
+    throw new Error(
+      `Local broad scan exceeded byte budget (${LOCAL_SCAN_MAX_TOTAL_BYTES}); narrow source.root or source.path.`,
+    );
+  }
+}
+function isExplicitRequestedPath(requestedRelativePath) {
+  return requestedRelativePath !== "" && requestedRelativePath !== ".";
+}
+function shouldSkipBroadDirectory(path, generatedPathGlobs) {
+  const name = path.split("/").at(-1)?.toLowerCase() ?? path.toLowerCase();
+  return BROAD_SKIP_DIRS.has(name) || matchesGeneratedPathHint(path, generatedPathGlobs);
+}
+function shouldSkipBroadFile(path, size, generatedPathGlobs) {
+  if (matchesGeneratedPathHint(path, generatedPathGlobs)) {
+    return true;
+  }
+  const name = path.split("/").at(-1)?.toLowerCase() ?? path.toLowerCase();
+  if (hasSecretFileName(name)) {
+    return true;
+  }
+  if (hasBroadSkippedFileExtension(name)) {
+    return true;
+  }
+  if (size > BROAD_SCAN_MAX_FILE_BYTES) {
+    return true;
+  }
+  if (name.endsWith(".min.js") || name.endsWith(".min.css") || name.endsWith(".map")) {
+    return true;
+  }
+  if (name.includes(".bundle.") || name.endsWith(".log")) {
+    return true;
+  }
+  if ((name.endsWith(".html") || name.endsWith(".json")) && size > 64_000) {
+    return true;
+  }
+  return false;
+}
+function matchesGeneratedPathHint(path, hints) {
+  if (hints.length === 0) {
+    return false;
+  }
+  const normalizedPath = normalizeRelativePath(path);
+  return hints.some((hint) => matchesGeneratedPathGlob(normalizedPath, hint));
+}
+function isSensitivePath(path) {
+  if (!path || path === ".") {
+    return false;
+  }
+  const segments = normalizeRelativePath(path)
+    .split("/")
+    .map((segment) => segment.toLowerCase());
+  return segments.some((segment) => SECRET_DIRS.has(segment)) || hasSecretFileName(segments.at(-1) ?? "");
+}
+function hasSecretFileName(name) {
+  return (
+    SECRET_FILE_NAMES.has(name) ||
+    name.startsWith(".env.") ||
+    name.endsWith(".pem") ||
+    name.endsWith(".key") ||
+    name.endsWith(".p12") ||
+    name.endsWith(".pfx")
+  );
+}
+function hasBroadSkippedFileExtension(name) {
+  for (const extension of BROAD_SKIP_FILE_EXTENSIONS) {
+    if (name.endsWith(extension)) {
+      return true;
+    }
+  }
+  return false;
+}
+function broadRootRejectionReason(rootRealPath) {
+  const normalized = normalizeAbsolutePath(rootRealPath);
+  const filesystemRoot = normalizeAbsolutePath(parse(rootRealPath).root);
+  if (normalized === filesystemRoot) {
+    return "Filesystem root scans are not allowed.";
+  }
+  const home = safeNormalizeAbsolutePath(homedir());
+  if (home && normalized === home) {
+    return "Home directory scans are not allowed.";
+  }
+  const temp = safeNormalizeAbsolutePath(tmpdir());
+  if (temp && normalized === temp) {
+    return "System temp directory scans are not allowed.";
+  }
+  const broadRoots = [
+    "/Applications",
+    "/Library",
+    "/System",
+    "/Users",
+    "/bin",
+    "/etc",
+    "/home",
+    "/opt",
+    "/private/tmp",
+    "/sbin",
+    "/tmp",
+    "/usr",
+    "/var",
+  ].map(normalizeAbsolutePath);
+  if (broadRoots.includes(normalized)) {
+    return "System or multi-user root scans are not allowed.";
+  }
+  if (SECRET_DIRS.has(basename(normalized).toLowerCase())) {
+    return "Credential/secret directory roots are not allowed.";
+  }
+  return null;
+}
+function safeNormalizeAbsolutePath(path) {
+  try {
+    return normalizeAbsolutePath(path);
+  } catch {
+    return null;
+  }
+}
+function isPathInsideRoot(root, absolutePath) {
+  const relativePath = relative(root, absolutePath);
+  return (
+    relativePath === "" ||
+    (!relativePath.startsWith("..") && !relativePath.startsWith("/") && !/^[A-Za-z]:/.test(relativePath))
+  );
+}
+function normalizeRelativePath(path) {
+  return path.split(/[\\/]+/).join("/");
+}
+function normalizeAbsolutePath(path) {
+  return normalizeRelativePath(resolve(path)).replace(/\/+$/, "") || "/";
+}

package/router/dist/processing/engine.d.ts

@@ -1,108 +1,132 @@
 import { type ProcessingReducerSelection } from "./reducers.js";
 import { type ProcessingScriptRequest, type ProcessingScriptResult } from "./scripts.js";
-import type { EvidenceLineage, EvidencePersistence, LocalFreeflowConfig, OutputStream, RecoveryHint, ScriptTransformConfig, SourceRef, VaultRetentionPolicy } from "../config/types.js";
+import type {
+  EvidenceLineage,
+  EvidencePersistence,
+  LocalFreeflowConfig,
+  OutputStream,
+  RecoveryHint,
+  ScriptTransformConfig,
+  SourceRef,
+  VaultRetentionPolicy,
+} from "../config/types.js";
 import type { ScriptSandboxAdapter } from "../sandbox/script-sandbox.js";
 export declare const PROCESSING_ENGINE_IMPLEMENTATION = "processing-engine-skeleton-v1";
 export interface ProcessingLimits {
-    maxSourceBytes: number;
-    maxVisibleBytes: number;
+  maxSourceBytes: number;
+  maxVisibleBytes: number;
 }
 export interface ProcessingEngineOptions {
-    sessionId?: string;
-    vaultRoot?: string;
-    vaultRetention?: VaultRetentionPolicy;
-    goal?: string;
-    limits?: Partial<ProcessingLimits>;
-    script?: ProcessingScriptRequest;
-    scriptTransform?: ScriptTransformConfig;
-    localConfig?: LocalFreeflowConfig;
-    scriptSandboxAdapters?: readonly ScriptSandboxAdapter[];
+  sessionId?: string;
+  vaultRoot?: string;
+  vaultRetention?: VaultRetentionPolicy;
+  goal?: string;
+  limits?: Partial<ProcessingLimits>;
+  script?: ProcessingScriptRequest;
+  scriptTransform?: ScriptTransformConfig;
+  localConfig?: LocalFreeflowConfig;
+  scriptSandboxAdapters?: readonly ScriptSandboxAdapter[];
 }
 export interface RepoFileProcessingSource {
-    kind: "repo-file";
-    root: string;
-    path: string;
+  kind: "repo-file";
+  root: string;
+  path: string;
 }
 export interface VaultOutputProcessingSource {
-    kind: "vault-output";
-    sessionId: string;
-    outputId: string;
-    stream?: OutputStream;
-    vaultRoot?: string;
+  kind: "vault-output";
+  sessionId: string;
+  outputId: string;
+  stream?: OutputStream;
+  vaultRoot?: string;
+}
+export interface LocalFileProcessingSource {
+  kind: "local-file";
+  root: string;
+  path: string;
 }
 export interface CapturedCommandOutputProcessingSource {
-    kind: "command-output";
-    stdout?: string;
-    stderr?: string;
-    combined?: string;
-    stream?: Exclude<OutputStream, "raw">;
-    outputId?: string;
-}
-export type ProcessingSourceInput = RepoFileProcessingSource | VaultOutputProcessingSource | CapturedCommandOutputProcessingSource;
+  kind: "command-output";
+  stdout?: string;
+  stderr?: string;
+  combined?: string;
+  stream?: Exclude<OutputStream, "raw">;
+  outputId?: string;
+}
+export type ProcessingSourceInput =
+  | RepoFileProcessingSource
+  | VaultOutputProcessingSource
+  | LocalFileProcessingSource
+  | CapturedCommandOutputProcessingSource;
 export interface ProcessingSourceStats {
-    bytes: number;
-    lines: number;
-    sha256: string;
+  bytes: number;
+  lines: number;
+  sha256: string;
 }
 export interface ProcessingSourceDescriptor {
-    kind: ProcessingSourceInput["kind"];
-    ref: SourceRef;
-    displayPath: string;
-    stream?: OutputStream;
+  kind: ProcessingSourceInput["kind"];
+  ref: SourceRef;
+  displayPath: string;
+  stream?: OutputStream;
 }
 export interface LoadedProcessingSource {
-    status: "ok";
-    source: ProcessingSourceDescriptor;
-    text: string;
-    stats: ProcessingSourceStats;
-    lineage?: EvidenceLineage;
-    persistence?: EvidencePersistence;
-    recovery?: RecoveryHint;
+  status: "ok";
+  source: ProcessingSourceDescriptor;
+  text: string;
+  stats: ProcessingSourceStats;
+  lineage?: EvidenceLineage;
+  persistence?: EvidencePersistence;
+  recovery?: RecoveryHint;
 }
 export interface BlockedProcessingSource {
-    status: "blocked";
-    source: ProcessingSourceDescriptor;
-    reason: string;
-    policy: "repo_containment" | "source_limit";
-    stats?: Pick<ProcessingSourceStats, "bytes">;
+  status: "blocked";
+  source: ProcessingSourceDescriptor;
+  reason: string;
+  policy: "repo_containment" | "source_limit";
+  stats?: Pick<ProcessingSourceStats, "bytes">;
 }
 export interface UnavailableProcessingSource {
-    status: "unavailable";
-    source: ProcessingSourceDescriptor;
-    reason: string;
+  status: "unavailable";
+  source: ProcessingSourceDescriptor;
+  reason: string;
 }
 export type ProcessingSourceLoadResult = LoadedProcessingSource | BlockedProcessingSource | UnavailableProcessingSource;
 export interface ProcessingFact {
-    name: string;
-    value: string | number | boolean;
+  name: string;
+  value: string | number | boolean;
 }
 export type ReducerSelectionResult = ProcessingReducerSelection;
 export type ScriptPolicySelectionResult = ProcessingScriptResult;
 export interface ProcessingResultBase {
-    implementation: typeof PROCESSING_ENGINE_IMPLEMENTATION;
-    source: ProcessingSourceDescriptor;
-    visibleText: string;
-    facts: ProcessingFact[];
-    reducer: ReducerSelectionResult;
-    script: ScriptPolicySelectionResult;
-    lineage?: EvidenceLineage;
-    persistence?: EvidencePersistence;
-    recovery?: RecoveryHint;
+  implementation: typeof PROCESSING_ENGINE_IMPLEMENTATION;
+  source: ProcessingSourceDescriptor;
+  visibleText: string;
+  facts: ProcessingFact[];
+  reducer: ReducerSelectionResult;
+  script: ScriptPolicySelectionResult;
+  lineage?: EvidenceLineage;
+  persistence?: EvidencePersistence;
+  recovery?: RecoveryHint;
 }
 export interface ProcessingOkResult extends ProcessingResultBase {
-    status: "ok";
-    stats: ProcessingSourceStats;
+  status: "ok";
+  stats: ProcessingSourceStats;
 }
 export interface ProcessingBlockedResult extends ProcessingResultBase {
-    status: "blocked";
-    reason: string;
-    policy: BlockedProcessingSource["policy"];
+  status: "blocked";
+  reason: string;
+  policy: BlockedProcessingSource["policy"];
 }
 export interface ProcessingUnavailableResult extends ProcessingResultBase {
-    status: "unavailable";
-    reason: string;
+  status: "unavailable";
+  reason: string;
 }
 export type ProcessingResult = ProcessingOkResult | ProcessingBlockedResult | ProcessingUnavailableResult;
-export declare function loadProcessingSource(source: ProcessingSourceInput, options?: ProcessingEngineOptions): Promise<ProcessingSourceLoadResult>;
-export declare function processSource(source: ProcessingSourceInput, options?: ProcessingEngineOptions): Promise<ProcessingResult>;
+export declare function loadProcessingSource(
+  source: ProcessingSourceInput,
+  options?: ProcessingEngineOptions,
+): Promise<ProcessingSourceLoadResult>;
+export declare function processSource(
+  source: ProcessingSourceInput,
+  options?: ProcessingEngineOptions,
+): Promise<ProcessingResult>;
 export declare function isProcessingPathInsideRoot(root: string, absolutePath: string): boolean;