@hassangameryt/freeflow 0.2.0 → 0.3.0

package/{plugins/freeflow/router/dist → router/dist/evidence}/evidence-search.js

@@ -192,8 +192,10 @@ function scoreCandidateChunk(chunk, queryTokens, normalizedQueryPhrase, stats, o
     const score = exactPhraseBoost(exactPhrase, chunk.file.path, queryTokens) +
         bm25Score(queryTokens, tokenCounts, chunkLength, stats) * 10 +
         coverageRatio(matchingTokens, queryTokens) * 120 +
+        contentCoverageBoost(matchingTokens, queryTokens, chunk.kind) +
         completeCoverageBoost(matchingTokens, queryTokens, chunk.kind) -
-        missingCoveragePenalty(matchingTokens, queryTokens, chunk.kind) +
+        missingCoveragePenalty(matchingTokens, queryTokens, chunk.kind) -
+        lowContentCoveragePenalty(matchingTokens, queryTokens, pathScore) +
         headingCoverageBoost(chunk.heading ?? "", queryTokens) +
         identifierBoost(chunk.text, queryTokens) +
         (matchingTokens.length >= 4 ? orderedPhraseBoost(chunkTokens, queryTokens) : 0) +
@@ -249,6 +251,13 @@ function bm25Score(queryTokens, tokenCounts, chunkLength, stats) {
 function coverageRatio(matchingTokens, queryTokens) {
     return queryTokens.length ? matchingTokens.length / queryTokens.length : 0;
 }
+function contentCoverageBoost(matchingTokens, queryTokens, chunkKind) {
+    if (queryTokens.length === 0) {
+        return 0;
+    }
+    const perTokenBoost = chunkKind === "window" ? 180 : 350;
+    return matchingTokens.length * perTokenBoost + coverageRatio(matchingTokens, queryTokens) * 100;
+}
 function completeCoverageBoost(matchingTokens, queryTokens, chunkKind) {
     if (queryTokens.length === 0 || matchingTokens.length !== queryTokens.length) {
         return 0;
@@ -264,6 +273,13 @@ function missingCoveragePenalty(matchingTokens, queryTokens, chunkKind) {
     }
     return (queryTokens.length - matchingTokens.length) * 90;
 }
+function lowContentCoveragePenalty(matchingTokens, queryTokens, pathScore) {
+    if (queryTokens.length < 4 || matchingTokens.length >= 2) {
+        return 0;
+    }
+    const pathOnlyPenalty = matchingTokens.length === 0 && pathScore > 0 ? 1_000 : 0;
+    return (2 - matchingTokens.length) * 600 + pathOnlyPenalty;
+}
 function exactPhraseBoost(exactPhrase, path, queryTokens) {
     if (!exactPhrase) {
         return 0;
@@ -333,7 +349,7 @@ function chunkKindBoost(kind) {
 function pathIntentBoost(path, queryTokens) {
     const pathTokens = new Set(tokenize(path));
     const coveredTokens = queryTokens.filter((token) => pathTokens.has(token)).length;
-    return coveredTokens * 28;
+    return coveredTokens * 140 + (coveredTokens >= 2 ? 600 : 0);
 }
 function sourceTestPrior(path, queryTokens) {
     const testPath = isTestPath(path);

package/{plugins/freeflow/router/dist → router/dist/evidence}/evidence.d.ts

@@ -1,4 +1,4 @@
-import type { ImportantLine } from "./types.js";
+import type { ImportantLine } from "../config/types.js";
 export type ImportantStream = ImportantLine["stream"];
 export interface LineEntry {
     line: string;

package/router/dist/evidence/failure-contracts.d.ts

@@ -0,0 +1,27 @@
+import type { EvidenceLineage, EvidencePacket, EvidencePersistence, FailureExecutionStatus, FailureRoutedResult, PreserveMode, RouterFailureKind, RoutingStatus, ToolStatus } from "../config/types.js";
+export type FailureOperationKind = "transform";
+export interface FailureResultOptions {
+    kind: RouterFailureKind;
+    operation: FailureOperationKind;
+    message: string;
+    preserve?: PreserveMode;
+    executionStatus?: FailureExecutionStatus;
+    toolStatus?: ToolStatus;
+    routingStatus?: RoutingStatus;
+    routingReason?: string;
+    persistence?: EvidencePersistence;
+    recordId?: string;
+    outputId?: string;
+    lineage?: EvidenceLineage;
+    evidence?: EvidencePacket[];
+    decisionSeed?: string;
+}
+export type TransformFailureOptions = Omit<FailureResultOptions, "kind" | "operation" | "executionStatus">;
+export type StorageFailureOptions = Omit<FailureResultOptions, "kind" | "operation" | "executionStatus">;
+export declare function storageFailure(options: StorageFailureOptions): FailureRoutedResult;
+export declare function scriptTransformDisabledFailure(options: TransformFailureOptions): FailureRoutedResult;
+export declare function transformAdapterUnavailableFailure(options: TransformFailureOptions): FailureRoutedResult;
+export declare function transformSourceUnavailableFailure(options: TransformFailureOptions): FailureRoutedResult;
+export declare function transformValidationFailure(options: TransformFailureOptions): FailureRoutedResult;
+export declare function transformExecutionFailure(options: TransformFailureOptions): FailureRoutedResult;
+export declare function createFailureResult(options: FailureResultOptions): FailureRoutedResult;

package/router/dist/evidence/failure-contracts.js

@@ -0,0 +1,160 @@
+import { createHash } from "node:crypto";
+const DEFAULT_PRESERVE = "important";
+export function storageFailure(options) {
+    return createFailureResult({
+        toolStatus: "error",
+        routingStatus: "failed",
+        ...options,
+        kind: "storage_failure",
+        operation: "transform",
+        executionStatus: "failed",
+        persistence: options.persistence ?? noPersistence(),
+    });
+}
+export function scriptTransformDisabledFailure(options) {
+    return createFailureResult({
+        ...options,
+        kind: "script_transform_disabled",
+        operation: "transform",
+        executionStatus: "unavailable",
+    });
+}
+export function transformAdapterUnavailableFailure(options) {
+    return createFailureResult({
+        ...options,
+        kind: "adapter_unavailable",
+        operation: "transform",
+        executionStatus: "unavailable",
+    });
+}
+export function transformSourceUnavailableFailure(options) {
+    return createFailureResult({
+        ...options,
+        kind: "transform_source_unavailable",
+        operation: "transform",
+        executionStatus: "unavailable",
+    });
+}
+export function transformValidationFailure(options) {
+    return createFailureResult({
+        ...options,
+        kind: "transform_validation_failure",
+        operation: "transform",
+        executionStatus: "rejected",
+    });
+}
+export function transformExecutionFailure(options) {
+    return createFailureResult({
+        ...options,
+        kind: "transform_execution_failure",
+        operation: "transform",
+        executionStatus: "failed",
+    });
+}
+export function createFailureResult(options) {
+    const preserve = options.preserve ?? DEFAULT_PRESERVE;
+    const persistence = normalizePersistence(options.persistence, options.outputId);
+    const routingStatus = options.routingStatus ?? "failed";
+    const recovery = recoveryHintForPersistence(persistence);
+    const executionStatus = options.executionStatus ?? defaultExecutionStatus(options.kind);
+    const result = {
+        toolStatus: options.toolStatus ?? "ok",
+        decisionId: failureDecisionId(options),
+        preserve,
+        routing: {
+            status: routingStatus,
+            route: "transform",
+            reason: options.routingReason ?? failureReason(options.kind, options.operation, options.message, persistence),
+        },
+        failure: {
+            kind: options.kind,
+            message: options.message,
+        },
+        persistence,
+        recovery,
+        transformExecution: executionFailure(executionStatus, options.kind, options.message),
+    };
+    if (options.recordId !== undefined) {
+        result.recordId = options.recordId;
+    }
+    if (options.outputId !== undefined) {
+        result.outputId = options.outputId;
+    }
+    if (options.lineage !== undefined) {
+        result.lineage = options.lineage;
+    }
+    if (options.evidence !== undefined) {
+        result.evidence = options.evidence;
+    }
+    return result;
+}
+function executionFailure(status, failureKind, message) {
+    return { status, failureKind, message };
+}
+function normalizePersistence(persistence, outputId) {
+    const candidate = persistence ?? noPersistence();
+    if (candidate.recoverability !== "exact" && candidate.recoverability !== "redacted") {
+        return candidate;
+    }
+    const recoveryOutputId = candidate.recoveryOutputId ?? candidate.outputId ?? outputId;
+    if (recoveryOutputId === undefined) {
+        return noPersistence();
+    }
+    return {
+        ...candidate,
+        recoveryOutputId,
+        ...(candidate.outputId !== undefined ? { outputId: candidate.outputId } : {}),
+    };
+}
+function noPersistence() {
+    return { status: "not_persisted", recoverability: "none" };
+}
+function recoveryHintForPersistence(persistence) {
+    const recoveryOutputId = persistence.recoveryOutputId ?? persistence.outputId;
+    if (persistence.recoverability === "exact" && recoveryOutputId !== undefined) {
+        return {
+            how: `Use freeflow_search with source.kind=vault and outputId=${recoveryOutputId} to recover exact transformed content.`,
+            outputId: recoveryOutputId,
+        };
+    }
+    if (persistence.recoverability === "redacted" && recoveryOutputId !== undefined) {
+        return {
+            how: `Use freeflow_search with source.kind=vault and outputId=${recoveryOutputId} to recover redacted persisted content. Exact raw recovery is unavailable.`,
+            outputId: recoveryOutputId,
+        };
+    }
+    if (persistence.recoverability === "metadata_only") {
+        return {
+            how: "Only metadata was persisted for this failure. No raw content stream is recoverable.",
+        };
+    }
+    return {
+        how: "No content was persisted for this failure. Recovery is unavailable.",
+    };
+}
+function defaultExecutionStatus(kind) {
+    if (kind === "adapter_unavailable" || kind === "transform_source_unavailable" || kind === "script_transform_disabled") {
+        return "unavailable";
+    }
+    if (kind === "transform_validation_failure") {
+        return "rejected";
+    }
+    return "failed";
+}
+function failureReason(kind, operation, message, persistence) {
+    return `${operation} failure kind=${kind}: ${message} persistence.status=${persistence.status}; recoverability=${persistence.recoverability}.`;
+}
+function failureDecisionId(options) {
+    return `ffdec_${hash(JSON.stringify([
+        "failure",
+        options.kind,
+        options.operation,
+        options.recordId ?? null,
+        options.outputId ?? null,
+        options.decisionSeed ?? null,
+        options.message,
+    ])).slice(0, 16)}`;
+}
+function hash(value) {
+    return createHash("sha256").update(value).digest("hex");
+}

package/router/dist/index.d.ts

@@ -0,0 +1,30 @@
+export * from "./tools/batch.js";
+export * from "./benchmarks/benchmarks.js";
+export * from "./benchmarks/command-benchmarks.js";
+export * from "./benchmarks/context-saving-baseline.js";
+export * from "./benchmarks/context-mode-normalized-benchmarks.js";
+export * from "./benchmarks/context-mode-real-deep-benchmark.js";
+export * from "./benchmarks/codex-qa-benchmarks.js";
+export * from "./experiments/local-index.js";
+export * from "./benchmarks/index-benchmarks.js";
+export * from "./benchmarks/vault-index-benchmarks.js";
+export * from "./config/config.js";
+export * from "./evidence/failure-contracts.js";
+export * from "./tools/search.js";
+export * from "./routing/observed-routing.js";
+export * from "./routing/observed-reducers.js";
+export * from "./processing/engine.js";
+export * from "./tools/run.js";
+export * from "./routing/run-filters.js";
+export * from "./sandbox/adapter-roots.js";
+export * from "./sandbox/quickjs-wasi-adapter.js";
+export * from "./sandbox/jq-wasm-adapter.js";
+export * from "./sandbox/eryx-python-adapter.js";
+export * from "./config/router-contract.js";
+export * from "./config/schema.js";
+export * from "./sandbox/script-sandbox.js";
+export * from "./benchmarks/storage-policy-benchmarks.js";
+export * from "./transform/engine.js";
+export * from "./config/types.js";
+export * from "./vault/vault.js";
+export * from "./vault/vault-index.js";

package/router/dist/index.js

@@ -0,0 +1,32 @@
+// Compatibility barrel for Pi extension and repo eval scripts.
+// Runtime code lives in responsibility-focused modules under router/src/*.
+export * from "./tools/batch.js";
+export * from "./benchmarks/benchmarks.js";
+export * from "./benchmarks/command-benchmarks.js";
+export * from "./benchmarks/context-saving-baseline.js";
+export * from "./benchmarks/context-mode-normalized-benchmarks.js";
+export * from "./benchmarks/context-mode-real-deep-benchmark.js";
+export * from "./benchmarks/codex-qa-benchmarks.js";
+export * from "./experiments/local-index.js";
+export * from "./benchmarks/index-benchmarks.js";
+export * from "./benchmarks/vault-index-benchmarks.js";
+export * from "./config/config.js";
+export * from "./evidence/failure-contracts.js";
+export * from "./tools/search.js";
+export * from "./routing/observed-routing.js";
+export * from "./routing/observed-reducers.js";
+export * from "./processing/engine.js";
+export * from "./tools/run.js";
+export * from "./routing/run-filters.js";
+export * from "./sandbox/adapter-roots.js";
+export * from "./sandbox/quickjs-wasi-adapter.js";
+export * from "./sandbox/jq-wasm-adapter.js";
+export * from "./sandbox/eryx-python-adapter.js";
+export * from "./config/router-contract.js";
+export * from "./config/schema.js";
+export * from "./sandbox/script-sandbox.js";
+export * from "./benchmarks/storage-policy-benchmarks.js";
+export * from "./transform/engine.js";
+export * from "./config/types.js";
+export * from "./vault/vault.js";
+export * from "./vault/vault-index.js";

package/router/dist/processing/engine.d.ts

@@ -0,0 +1,108 @@
+import { type ProcessingReducerSelection } from "./reducers.js";
+import { type ProcessingScriptRequest, type ProcessingScriptResult } from "./scripts.js";
+import type { EvidenceLineage, EvidencePersistence, LocalFreeflowConfig, OutputStream, RecoveryHint, ScriptTransformConfig, SourceRef, VaultRetentionPolicy } from "../config/types.js";
+import type { ScriptSandboxAdapter } from "../sandbox/script-sandbox.js";
+export declare const PROCESSING_ENGINE_IMPLEMENTATION = "processing-engine-skeleton-v1";
+export interface ProcessingLimits {
+    maxSourceBytes: number;
+    maxVisibleBytes: number;
+}
+export interface ProcessingEngineOptions {
+    sessionId?: string;
+    vaultRoot?: string;
+    vaultRetention?: VaultRetentionPolicy;
+    goal?: string;
+    limits?: Partial<ProcessingLimits>;
+    script?: ProcessingScriptRequest;
+    scriptTransform?: ScriptTransformConfig;
+    localConfig?: LocalFreeflowConfig;
+    scriptSandboxAdapters?: readonly ScriptSandboxAdapter[];
+}
+export interface RepoFileProcessingSource {
+    kind: "repo-file";
+    root: string;
+    path: string;
+}
+export interface VaultOutputProcessingSource {
+    kind: "vault-output";
+    sessionId: string;
+    outputId: string;
+    stream?: OutputStream;
+    vaultRoot?: string;
+}
+export interface CapturedCommandOutputProcessingSource {
+    kind: "command-output";
+    stdout?: string;
+    stderr?: string;
+    combined?: string;
+    stream?: Exclude<OutputStream, "raw">;
+    outputId?: string;
+}
+export type ProcessingSourceInput = RepoFileProcessingSource | VaultOutputProcessingSource | CapturedCommandOutputProcessingSource;
+export interface ProcessingSourceStats {
+    bytes: number;
+    lines: number;
+    sha256: string;
+}
+export interface ProcessingSourceDescriptor {
+    kind: ProcessingSourceInput["kind"];
+    ref: SourceRef;
+    displayPath: string;
+    stream?: OutputStream;
+}
+export interface LoadedProcessingSource {
+    status: "ok";
+    source: ProcessingSourceDescriptor;
+    text: string;
+    stats: ProcessingSourceStats;
+    lineage?: EvidenceLineage;
+    persistence?: EvidencePersistence;
+    recovery?: RecoveryHint;
+}
+export interface BlockedProcessingSource {
+    status: "blocked";
+    source: ProcessingSourceDescriptor;
+    reason: string;
+    policy: "repo_containment" | "source_limit";
+    stats?: Pick<ProcessingSourceStats, "bytes">;
+}
+export interface UnavailableProcessingSource {
+    status: "unavailable";
+    source: ProcessingSourceDescriptor;
+    reason: string;
+}
+export type ProcessingSourceLoadResult = LoadedProcessingSource | BlockedProcessingSource | UnavailableProcessingSource;
+export interface ProcessingFact {
+    name: string;
+    value: string | number | boolean;
+}
+export type ReducerSelectionResult = ProcessingReducerSelection;
+export type ScriptPolicySelectionResult = ProcessingScriptResult;
+export interface ProcessingResultBase {
+    implementation: typeof PROCESSING_ENGINE_IMPLEMENTATION;
+    source: ProcessingSourceDescriptor;
+    visibleText: string;
+    facts: ProcessingFact[];
+    reducer: ReducerSelectionResult;
+    script: ScriptPolicySelectionResult;
+    lineage?: EvidenceLineage;
+    persistence?: EvidencePersistence;
+    recovery?: RecoveryHint;
+}
+export interface ProcessingOkResult extends ProcessingResultBase {
+    status: "ok";
+    stats: ProcessingSourceStats;
+}
+export interface ProcessingBlockedResult extends ProcessingResultBase {
+    status: "blocked";
+    reason: string;
+    policy: BlockedProcessingSource["policy"];
+}
+export interface ProcessingUnavailableResult extends ProcessingResultBase {
+    status: "unavailable";
+    reason: string;
+}
+export type ProcessingResult = ProcessingOkResult | ProcessingBlockedResult | ProcessingUnavailableResult;
+export declare function loadProcessingSource(source: ProcessingSourceInput, options?: ProcessingEngineOptions): Promise<ProcessingSourceLoadResult>;
+export declare function processSource(source: ProcessingSourceInput, options?: ProcessingEngineOptions): Promise<ProcessingResult>;
+export declare function isProcessingPathInsideRoot(root: string, absolutePath: string): boolean;