@cleocode/lafs-protocol 0.5.0 → 1.0.0

package/dist/src/tokenEstimator.d.ts

@@ -0,0 +1,87 @@
+/**
+ * LAFS Token Estimator
+ *
+ * Provides character-based token estimation for LAFS envelopes and JSON payloads.
+ * Uses the approximation: 1 token ≈ 4 characters.
+ * Properly handles nested objects, arrays, Unicode graphemes, and circular references.
+ */
+export interface TokenEstimatorOptions {
+    /**
+     * Characters per token ratio (default: 4)
+     */
+    charsPerToken?: number;
+    /**
+     * Maximum depth to traverse for circular reference detection (default: 100)
+     */
+    maxDepth?: number;
+    /**
+     * Maximum string length to process for Unicode grapheme counting (default: 100000)
+     */
+    maxStringLength?: number;
+}
+/**
+ * TokenEstimator provides character-based token counting for JSON payloads.
+ *
+ * Algorithm:
+ * 1. Serialize value to JSON (handling circular refs)
+ * 2. Count Unicode graphemes (not bytes)
+ * 3. Divide by charsPerToken ratio (default 4)
+ * 4. Add overhead for structural characters
+ */
+export declare class TokenEstimator {
+    private options;
+    constructor(options?: TokenEstimatorOptions);
+    /**
+     * Estimate tokens for any JavaScript value.
+     * Handles circular references, nested objects, arrays, and Unicode.
+     *
+     * @param value - Any value to estimate
+     * @returns Estimated token count
+     */
+    estimate(value: unknown): number;
+    /**
+     * Estimate tokens from a JSON string.
+     * More efficient if you already have the JSON string.
+     *
+     * @param json - JSON string to estimate
+     * @returns Estimated token count
+     */
+    estimateJSON(json: string): number;
+    /**
+     * Internal recursive estimation with circular reference tracking.
+     */
+    private estimateWithTracking;
+    /**
+     * Estimate tokens for an array.
+     */
+    private estimateArray;
+    /**
+     * Estimate tokens for a plain object.
+     */
+    private estimateObject;
+    /**
+     * Check if a value can be safely serialized (no circular refs).
+     */
+    canSerialize(value: unknown): boolean;
+    /**
+     * Serialize value to JSON with circular reference handling.
+     * Circular refs are replaced with "[Circular]".
+     */
+    safeStringify(value: unknown): string;
+    /**
+     * Create a safe copy of a value with circular refs removed.
+     */
+    safeCopy<T>(value: T): T;
+}
+/**
+ * Global token estimator instance with default settings.
+ */
+export declare const defaultEstimator: TokenEstimator;
+/**
+ * Convenience function to estimate tokens for a value.
+ */
+export declare function estimateTokens(value: unknown, options?: TokenEstimatorOptions): number;
+/**
+ * Convenience function to estimate tokens from a JSON string.
+ */
+export declare function estimateTokensJSON(json: string, options?: TokenEstimatorOptions): number;

package/dist/src/tokenEstimator.js

@@ -0,0 +1,238 @@
+/**
+ * LAFS Token Estimator
+ *
+ * Provides character-based token estimation for LAFS envelopes and JSON payloads.
+ * Uses the approximation: 1 token ≈ 4 characters.
+ * Properly handles nested objects, arrays, Unicode graphemes, and circular references.
+ */
+/**
+ * Counts Unicode graphemes in a string using Intl.Segmenter when available.
+ * Falls back to character counting for environments without Intl.Segmenter.
+ */
+function countGraphemes(str) {
+    // Use Intl.Segmenter for proper grapheme counting (Node.js 16+, modern browsers)
+    if (typeof Intl !== 'undefined' && 'Segmenter' in Intl) {
+        // @ts-ignore - Intl.Segmenter may not be in all TypeScript lib versions
+        const segmenter = new Intl.Segmenter('en', { granularity: 'grapheme' });
+        // @ts-ignore
+        return Array.from(segmenter.segment(str)).length;
+    }
+    // Fallback: count code points using spread operator (handles surrogate pairs)
+    return [...str].length;
+}
+/**
+ * Default options for token estimation
+ */
+const DEFAULT_OPTIONS = {
+    charsPerToken: 4,
+    maxDepth: 100,
+    maxStringLength: 100000,
+};
+/**
+ * TokenEstimator provides character-based token counting for JSON payloads.
+ *
+ * Algorithm:
+ * 1. Serialize value to JSON (handling circular refs)
+ * 2. Count Unicode graphemes (not bytes)
+ * 3. Divide by charsPerToken ratio (default 4)
+ * 4. Add overhead for structural characters
+ */
+export class TokenEstimator {
+    options;
+    constructor(options = {}) {
+        this.options = { ...DEFAULT_OPTIONS, ...options };
+    }
+    /**
+     * Estimate tokens for any JavaScript value.
+     * Handles circular references, nested objects, arrays, and Unicode.
+     *
+     * @param value - Any value to estimate
+     * @returns Estimated token count
+     */
+    estimate(value) {
+        return this.estimateWithTracking(value, new WeakSet(), 0);
+    }
+    /**
+     * Estimate tokens from a JSON string.
+     * More efficient if you already have the JSON string.
+     *
+     * @param json - JSON string to estimate
+     * @returns Estimated token count
+     */
+    estimateJSON(json) {
+        // Count graphemes in the JSON string
+        const graphemes = countGraphemes(json);
+        // Add overhead for JSON structure (brackets, quotes, colons, etc.)
+        const structuralOverhead = Math.ceil(graphemes * 0.1);
+        return Math.ceil((graphemes + structuralOverhead) / this.options.charsPerToken);
+    }
+    /**
+     * Internal recursive estimation with circular reference tracking.
+     */
+    estimateWithTracking(value, seen, depth) {
+        // Prevent infinite recursion
+        if (depth > this.options.maxDepth) {
+            return 1; // Minimal cost for max depth exceeded
+        }
+        // Handle null
+        if (value === null) {
+            return 1; // "null" = 4 chars / 4 = 1 token
+        }
+        // Handle undefined
+        if (value === undefined) {
+            return 1;
+        }
+        // Handle primitives
+        const type = typeof value;
+        if (type === 'boolean') {
+            return value ? 1 : 1; // "true" or "false" ≈ 1 token
+        }
+        if (type === 'number') {
+            const str = String(value);
+            return Math.ceil(countGraphemes(str) / this.options.charsPerToken);
+        }
+        if (type === 'string') {
+            const str = value;
+            // Limit string length to prevent performance issues
+            const truncated = str.length > this.options.maxStringLength
+                ? str.slice(0, this.options.maxStringLength) + '…'
+                : str;
+            const graphemes = countGraphemes(truncated);
+            // Add 2 for quotes
+            return Math.ceil((graphemes + 2) / this.options.charsPerToken);
+        }
+        // Handle objects and arrays
+        if (type === 'object') {
+            const obj = value;
+            // Check for circular reference
+            if (seen.has(obj)) {
+                return 1; // Minimal cost for circular ref placeholder
+            }
+            seen.add(obj);
+            try {
+                if (Array.isArray(obj)) {
+                    return this.estimateArray(obj, seen, depth);
+                }
+                return this.estimateObject(obj, seen, depth);
+            }
+            finally {
+                seen.delete(obj);
+            }
+        }
+        // Handle symbols, functions, etc.
+        return 1;
+    }
+    /**
+     * Estimate tokens for an array.
+     */
+    estimateArray(arr, seen, depth) {
+        let tokens = 1; // Opening bracket [ (already counted as structural)
+        for (let i = 0; i < arr.length; i++) {
+            tokens += this.estimateWithTracking(arr[i], seen, depth + 1);
+            // Add comma separator (except for last element)
+            if (i < arr.length - 1) {
+                tokens += 1; // comma + space ≈ 2 chars / 4 = 0.5, round up to 1
+            }
+        }
+        tokens += 1; // Closing bracket ]
+        return tokens;
+    }
+    /**
+     * Estimate tokens for a plain object.
+     */
+    estimateObject(obj, seen, depth) {
+        let tokens = 1; // Opening brace {
+        const keys = Object.keys(obj);
+        for (let i = 0; i < keys.length; i++) {
+            const key = keys[i];
+            const value = obj[key];
+            // Estimate key (with quotes)
+            tokens += Math.ceil((countGraphemes(key) + 2) / this.options.charsPerToken);
+            // Colon separator
+            tokens += 1; // " : " ≈ 3 chars / 4 = 0.75, round up to 1
+            // Estimate value
+            tokens += this.estimateWithTracking(value, seen, depth + 1);
+            // Comma separator (except for last property)
+            if (i < keys.length - 1) {
+                tokens += 1; // comma + space ≈ 2 chars / 4 = 0.5, round up to 1
+            }
+        }
+        tokens += 1; // Closing brace }
+        return tokens;
+    }
+    /**
+     * Check if a value can be safely serialized (no circular refs).
+     */
+    canSerialize(value) {
+        try {
+            JSON.stringify(value);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Serialize value to JSON with circular reference handling.
+     * Circular refs are replaced with "[Circular]".
+     */
+    safeStringify(value) {
+        const seen = new WeakSet();
+        return JSON.stringify(value, (key, val) => {
+            if (typeof val === 'object' && val !== null) {
+                if (seen.has(val)) {
+                    return '[Circular]';
+                }
+                seen.add(val);
+            }
+            return val;
+        });
+    }
+    /**
+     * Create a safe copy of a value with circular refs removed.
+     */
+    safeCopy(value) {
+        const seen = new WeakSet();
+        function clone(val) {
+            if (val === null || typeof val !== 'object') {
+                return val;
+            }
+            if (seen.has(val)) {
+                return '[Circular]';
+            }
+            seen.add(val);
+            try {
+                if (Array.isArray(val)) {
+                    return val.map(clone);
+                }
+                const result = {};
+                for (const [k, v] of Object.entries(val)) {
+                    result[k] = clone(v);
+                }
+                return result;
+            }
+            finally {
+                seen.delete(val);
+            }
+        }
+        return clone(value);
+    }
+}
+/**
+ * Global token estimator instance with default settings.
+ */
+export const defaultEstimator = new TokenEstimator();
+/**
+ * Convenience function to estimate tokens for a value.
+ */
+export function estimateTokens(value, options) {
+    const estimator = options ? new TokenEstimator(options) : defaultEstimator;
+    return estimator.estimate(value);
+}
+/**
+ * Convenience function to estimate tokens from a JSON string.
+ */
+export function estimateTokensJSON(json, options) {
+    const estimator = options ? new TokenEstimator(options) : defaultEstimator;
+    return estimator.estimateJSON(json);
+}

package/dist/src/types.d.ts

@@ -85,3 +85,28 @@ export interface ConformanceReport {
         detail?: string;
     }>;
 }
+export type BudgetEnforcementOptions = {
+    truncateOnExceed?: boolean;
+    onBudgetExceeded?: (estimated: number, budget: number) => void;
+};
+export interface TokenEstimate {
+    estimated: number;
+    truncated?: boolean;
+    originalEstimate?: number;
+}
+export interface LAFSMetaWithBudget extends LAFSMeta {
+    _tokenEstimate?: TokenEstimate;
+}
+export interface LAFSEnvelopeWithBudget extends Omit<LAFSEnvelope, '_meta'> {
+    _meta: LAFSMetaWithBudget;
+}
+export type MiddlewareFunction = (envelope: LAFSEnvelope) => LAFSEnvelope | Promise<LAFSEnvelope>;
+export type NextFunction = () => LAFSEnvelope | Promise<LAFSEnvelope>;
+export type BudgetMiddleware = (envelope: LAFSEnvelope, next: NextFunction) => Promise<LAFSEnvelope> | LAFSEnvelope;
+export interface BudgetEnforcementResult {
+    envelope: LAFSEnvelope;
+    withinBudget: boolean;
+    estimatedTokens: number;
+    budget: number;
+    truncated: boolean;
+}

package/lafs.md

@@ -176,6 +176,56 @@ Rules:
 - Decisions affecting output MUST be represented in ledger state.
 - Missing required context for a mutating step MUST fail with structured error.
 
+### 8.1 Context Retrieval
+
+Agents MAY retrieve context ledger state via `GET /_lafs/context/{ledgerId}` with projection modes.
+
+#### 8.1.1 Projection Modes
+
+**Full Mode (`mode=full`):**
+Returns complete ledger including all entries.
+- Use for: Initial loads, recovery scenarios
+- Supports: Offset-based pagination
+- Response includes: All ledger fields
+
+**Delta Mode (`mode=delta&sinceVersion=N`):**
+Returns only entries added since version N.
+- Use for: Active workflows (efficient sync)
+- Response includes:
+```json
+{
+  "ledgerId": "ctx_abc123",
+  "mode": "delta",
+  "fromVersion": 10,
+  "toVersion": 15,
+  "entries": [/* new entries only */],
+  "removedConstraints": [/* constraints no longer active */],
+  "checksum": "sha256:..."
+}
+```
+
+**Summary Mode (`mode=summary`):**
+Returns checksum and version for validation.
+- Use for: Quick sync validation
+- Response includes only: `ledgerId`, `version`, `checksum`, `entryCount`
+
+#### 8.1.2 Query Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `mode` | enum | `full`, `delta`, `summary` |
+| `sinceVersion` | integer | For delta mode: return entries after this version |
+| `filterByOperation` | string[] | Filter entries by operation name(s) |
+| `limit` | integer | Max entries (1-1000, default 100) |
+| `includeChecksum` | boolean | Include integrity checksum (default true) |
+
+#### 8.1.3 Agent Guidance
+
+- **Initial load**: Use `mode=full` once
+- **Active workflows**: Use `mode=delta` with last known version
+- **Validation**: Use `mode=summary` to verify sync state
+- **Default recommendation**: `delta` mode for agent-optimal behavior
+
 ---
 
 ## 9. MVI and Progressive Disclosure
@@ -212,6 +262,120 @@ Clients MAY request expanded/nested data via the `_expand` request parameter.
 - Pagination mode (offset or cursor) MUST be documented.
 - Mixed pagination modes in one request MUST fail validation.
 
+### 9.5 Token Budget Signaling
+
+Token budget signaling enables clients to declare resource constraints that servers MUST respect when generating responses. This mechanism prevents context window overflow in LLM-driven workflows.
+
+#### 9.5.1 Budget Declaration (`_budget`)
+
+Clients MAY declare resource constraints via the `_budget` request parameter:
+
+```json
+{
+  "_budget": {
+    "maxTokens": 4000,
+    "maxBytes": 32768,
+    "maxItems": 100
+  }
+}
+```
+
+**Fields:**
+- `maxTokens` (integer) - Maximum approximate tokens
+- `maxBytes` (integer) - Maximum byte size
+- `maxItems` (integer) - Maximum items in lists
+
+**Constraints:**
+- At least one field MUST be present
+- All values MUST be positive integers
+- Servers MAY reject budgets exceeding implementation limits
+
+#### 9.5.2 Server Behavior
+
+Servers MUST:
+1. Parse `_budget` from incoming requests
+2. Estimate/measure response size
+3. Return response within budget OR fail with `E_MVI_BUDGET_EXCEEDED`
+
+Servers MAY truncate responses using:
+- **Depth-first**: Remove deepest nested fields
+- **Field priority**: Remove non-essential fields first
+- **Hybrid**: Combine both strategies
+
+When truncation occurs, servers MUST include:
+```json
+{
+  "_meta": {
+    "warnings": [{
+      "code": "E_MVI_BUDGET_TRUNCATED",
+      "message": "Response truncated to fit token budget"
+    }],
+    "_tokenEstimate": {
+      "estimated": 2847,
+      "budget": 4000,
+      "method": "character_based"
+    }
+  }
+}
+```
+
+#### 9.5.3 Error Specification
+
+**E_MVI_BUDGET_EXCEEDED:**
+- **Category:** `VALIDATION`
+- **Retryable:** `true`
+- **Details:** `estimatedTokens`, `budget`, `excessTokens`, `constraint`
+
+```json
+{
+  "error": {
+    "code": "E_MVI_BUDGET_EXCEEDED",
+    "message": "Response exceeds declared token budget",
+    "category": "VALIDATION",
+    "retryable": true,
+    "details": {
+      "estimatedTokens": 5234,
+      "budget": 4000,
+      "excessTokens": 1234,
+      "constraint": "maxTokens"
+    }
+  }
+}
+```
+
+#### 9.5.4 Token Estimation Algorithm (Normative)
+
+Servers MUST implement this algorithm or equivalent (within +/- 10%):
+
+```
+FUNCTION estimate_tokens(value, depth = 0):
+    IF depth > 20: RETURN INFINITY
+    IF value IS null: RETURN 1
+    IF value IS boolean: RETURN 1
+    IF value IS number: RETURN max(1, len(stringify(value)) / 4)
+    IF value IS string:
+        graphemes = count_grapheme_clusters(value)
+        RETURN max(1, graphemes / 4.0)
+    IF value IS array:
+        tokens = 2  // []
+        FOR item IN value:
+            tokens += estimate_tokens(item, depth + 1) + 1
+        RETURN tokens
+    IF value IS object:
+        tokens = 2  // {}
+        FOR key, val IN value:
+            tokens += estimate_tokens(key, depth + 1)
+            tokens += 2  // : and ,
+            tokens += estimate_tokens(val, depth + 1)
+        RETURN tokens
+```
+
+**Requirements:**
+- Count grapheme clusters (not bytes) for unicode
+- Enforce max depth of 20
+- Handle circular references
+- Complete within 10ms for 100KB payloads
+
 ---
 
 ## 10. Strictness

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cleocode/lafs-protocol",
-  "version": "0.5.0",
+  "version": "1.0.0",
   "private": false,
   "type": "module",
   "description": "LLM-Agent-First Specification schemas and conformance tooling",
@@ -50,13 +50,18 @@
   ],
   "license": "MIT",
   "devDependencies": {
+    "@modelcontextprotocol/sdk": "^1.26.0",
+    "@types/express": "^5.0.6",
     "@types/node": "^24.3.0",
+    "@types/supertest": "^6.0.3",
+    "supertest": "^7.2.2",
     "tsx": "^4.20.5",
     "typescript": "^5.9.2",
     "vitest": "^2.1.9"
   },
   "dependencies": {
-    "ajv": "^8.17.1",
-    "ajv-formats": "^3.0.1"
+    "ajv": "^8.18.0",
+    "ajv-formats": "^3.0.1",
+    "express": "^5.2.1"
   }
 }

package/schemas/v1/discovery.schema.json

@@ -0,0 +1,132 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://lafs.dev/schemas/v1/discovery.schema.json",
+  "title": "LAFS Discovery Document",
+  "description": "Schema for LAFS agent discovery documents served at /.well-known/lafs.json",
+  "type": "object",
+  "required": [
+    "$schema",
+    "lafs_version",
+    "service",
+    "capabilities",
+    "endpoints"
+  ],
+  "properties": {
+    "$schema": {
+      "type": "string",
+      "format": "uri",
+      "description": "URL of the schema this document conforms to"
+    },
+    "lafs_version": {
+      "type": "string",
+      "pattern": "^\\d+\\.\\d+\\.\\d+$",
+      "description": "LAFS protocol version (semantic versioning)"
+    },
+    "service": {
+      "type": "object",
+      "description": "Service identification and metadata",
+      "required": [
+        "name",
+        "version"
+      ],
+      "properties": {
+        "name": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 100,
+          "description": "Unique service name (kebab-case recommended)"
+        },
+        "version": {
+          "type": "string",
+          "pattern": "^\\d+\\.\\d+\\.\\d+$",
+          "description": "Service version (semantic versioning)"
+        },
+        "description": {
+          "type": "string",
+          "maxLength": 500,
+          "description": "Human-readable service description"
+        }
+      },
+      "additionalProperties": false
+    },
+    "capabilities": {
+      "type": "array",
+      "description": "List of LAFS capabilities this service provides",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "required": [
+          "name",
+          "version",
+          "operations"
+        ],
+        "properties": {
+          "name": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 100,
+            "description": "Capability identifier (kebab-case recommended)"
+          },
+          "version": {
+            "type": "string",
+            "pattern": "^\\d+\\.\\d+\\.\\d+$",
+            "description": "Capability version (semantic versioning)"
+          },
+          "description": {
+            "type": "string",
+            "maxLength": 500,
+            "description": "Human-readable capability description"
+          },
+          "operations": {
+            "type": "array",
+            "description": "List of operations this capability supports",
+            "minItems": 1,
+            "items": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 50
+            }
+          },
+          "optional": {
+            "type": "boolean",
+            "default": false,
+            "description": "Whether this capability is optional for clients"
+          }
+        },
+        "additionalProperties": false
+      }
+    },
+    "endpoints": {
+      "type": "object",
+      "description": "URL endpoints for LAFS operations",
+      "required": [
+        "envelope",
+        "discovery"
+      ],
+      "properties": {
+        "envelope": {
+          "type": "string",
+          "minLength": 1,
+          "description": "URL for envelope submission (POST)"
+        },
+        "context": {
+          "type": "string",
+          "minLength": 1,
+          "description": "URL for context ledger operations"
+        },
+        "discovery": {
+          "type": "string",
+          "minLength": 1,
+          "description": "URL of this discovery document"
+        }
+      },
+      "additionalProperties": false
+    },
+    "extensions": {
+      "type": "object",
+      "description": "Extension fields for vendor-specific metadata",
+      "additionalProperties": true
+    }
+  },
+  "additionalProperties": false
+}