@aigne/ash 0.0.1

package/src/reference.test.ts

@@ -0,0 +1,166 @@
+import { describe, it, expect } from "vitest";
+import { ASH_REFERENCE } from "./reference.js";
+import { compileSource } from "./compiler.js";
+
+describe("ASH_REFERENCE", () => {
+  it("is a non-empty string", () => {
+    expect(typeof ASH_REFERENCE).toBe("string");
+    expect(ASH_REFERENCE.length).toBeGreaterThan(500);
+  });
+
+  // ── All builtin commands documented ──
+
+  const commands = [
+    "find",
+    "where",
+    "map",
+    "save",
+    "publish",
+    "tee",
+    "fanout",
+    "output",
+    "input",
+    "count",
+    "group-by",
+    "action",
+    "route",
+    "lookup",
+  ];
+
+  for (const cmd of commands) {
+    it(`documents command: ${cmd}`, () => {
+      expect(ASH_REFERENCE).toContain(cmd);
+    });
+  }
+
+  // ── All annotations documented ──
+
+  const annotations = ["@approval", "@retry", "@timeout", "@readonly", "@on_error"];
+
+  for (const ann of annotations) {
+    it(`documents annotation: ${ann}`, () => {
+      expect(ASH_REFERENCE).toContain(ann);
+    });
+  }
+
+  // ── Key language features documented ──
+
+  it("documents variables (let + $ref)", () => {
+    expect(ASH_REFERENCE).toContain("let ");
+    expect(ASH_REFERENCE).toMatch(/\$\w+/); // $variable reference
+  });
+
+  it("documents comments (#)", () => {
+    expect(ASH_REFERENCE).toContain("#");
+  });
+
+  it("documents jobs", () => {
+    expect(ASH_REFERENCE).toContain("job ");
+  });
+
+  it("documents pipe operator", () => {
+    expect(ASH_REFERENCE).toContain("|");
+  });
+
+  it("documents comparison operators", () => {
+    expect(ASH_REFERENCE).toContain("==");
+    expect(ASH_REFERENCE).toContain("!=");
+    expect(ASH_REFERENCE).toContain(">");
+    expect(ASH_REFERENCE).toContain("<");
+  });
+
+  it("documents map object transform syntax { key field }", () => {
+    expect(ASH_REFERENCE).toMatch(/map\s*\{/);
+  });
+
+  it("documents find...where inline query", () => {
+    expect(ASH_REFERENCE).toMatch(/find\s+\/\S+\s+where/);
+  });
+
+  // ── Type system documented ──
+
+  it("documents stream types", () => {
+    expect(ASH_REFERENCE).toContain("object_stream");
+  });
+
+  it("explains that save/publish produce no output", () => {
+    // Should mention that save and publish are terminal (output none)
+    expect(ASH_REFERENCE).toMatch(/save.*none|none.*save/i);
+  });
+
+  // ── Contains at least one complete example ──
+
+  it("contains a complete working example in a code block", () => {
+    // Extract code blocks from the reference
+    const codeBlocks = ASH_REFERENCE.match(/```ash\n([\s\S]*?)```/g);
+    expect(codeBlocks).not.toBeNull();
+    expect(codeBlocks!.length).toBeGreaterThan(0);
+  });
+
+  it("all code block examples compile without errors", () => {
+    const codeBlockMatches = ASH_REFERENCE.matchAll(/```ash\n([\s\S]*?)```/g);
+    let blockCount = 0;
+    for (const match of codeBlockMatches) {
+      const source = match[1].trim();
+      if (source.length === 0) continue;
+      blockCount++;
+      const result = compileSource(source);
+      const errors = result.diagnostics.filter(d => d.severity !== "warning");
+      expect(errors, `Code block should compile: ${source.slice(0, 80)}...`).toEqual([]);
+    }
+    expect(blockCount).toBeGreaterThan(0);
+  });
+
+  // ── v3 features documented ──
+
+  it("documents action /path command", () => {
+    expect(ASH_REFERENCE).toMatch(/action\s+\/\S+/);
+  });
+
+  it("documents route { ... } command", () => {
+    expect(ASH_REFERENCE).toMatch(/route\b/);
+  });
+
+  it("documents lookup /path on key command", () => {
+    expect(ASH_REFERENCE).toMatch(/lookup\s+\/\S+\s+on\s+\w+/);
+  });
+
+  it("documents expression syntax (arithmetic + $var)", () => {
+    expect(ASH_REFERENCE).toMatch(/\*\s*\d|\d\s*\*/); // multiplication
+    expect(ASH_REFERENCE).toMatch(/\$\w+/); // $variable in expressions
+  });
+
+  it("documents map { key: expr } colon syntax", () => {
+    expect(ASH_REFERENCE).toMatch(/map\s*\{[^}]*:/); // colon inside map braces
+  });
+
+  it("documents param name = default declaration", () => {
+    expect(ASH_REFERENCE).toMatch(/param\s+\w+\s*=/);
+  });
+
+  it("documents on /path:event trigger declaration", () => {
+    expect(ASH_REFERENCE).toMatch(/on\s+\/\S+:\w+/);
+  });
+
+  it("documents @on_error annotation", () => {
+    expect(ASH_REFERENCE).toContain("@on_error");
+  });
+
+  it("documents runtime let with expressions", () => {
+    // let with pipeline assignment
+    expect(ASH_REFERENCE).toMatch(/let\s+\w+\s*=\s*find/);
+  });
+
+  // ── No stale references ──
+
+  it("does not reference removed 'echo' command", () => {
+    // echo was removed — reference must not mention it as a command
+    expect(ASH_REFERENCE).not.toMatch(/\becho\b/i);
+  });
+
+  // ── Reasonable size ──
+
+  it("is concise enough for LLM context (under 6000 chars)", () => {
+    expect(ASH_REFERENCE.length).toBeLessThan(6000);
+  });
+});

package/src/reference.ts

@@ -0,0 +1,125 @@
+/**
+ * ASH language reference — designed for LLM consumption.
+ *
+ * Exported as a string constant so any consumer (AFS provider, AI dev loop,
+ * CLI help) can use the same single source of truth.
+ */
+export const ASH_REFERENCE = `# ASH Language Reference
+
+ASH is a deterministic pipeline DSL. No imports, no eval, no control flow — compiler-safe by design.
+
+## Structure
+
+\`\`\`ash
+# Comments start with #
+param threshold = 70
+
+let total = find /data/users | count
+
+output "Starting ETL"
+
+@caps(read /data/* write /data/*)
+job extract {
+  find /data/users where active == true
+  | where score > $threshold
+  | map { fullName: name + " " + surname, score: score * 2 + $bonus }
+  | tee /data/backup
+  | save /data/clean
+}
+\`\`\`
+
+**Top-level statements**: \`param\`, \`let\`, \`output\`, \`job\`
+**Pipes**: stages separated by \`|\`
+
+## Commands
+
+| Command | Input | Output | Description |
+|---------|-------|--------|-------------|
+| \`find /path\` | none | object_stream | Read records from path. Each record contains \`path\`, \`name\`, \`kind\` + provider-specific content fields (e.g. \`battery_level\`, \`state\`, \`label\`). |
+| \`find /path where f == v\` | none | object_stream | Read with inline filter (query pushdown). Same output schema as \`find\`. |
+| \`where field op value\` | object_stream | object_stream | Filter records (\`==\` \`!=\` \`>\` \`<\` \`>=\` \`<=\`) |
+| \`map field\` | object_stream | object_stream | Extract single field |
+| \`map { out1 in1, out2 in2 }\` | object_stream | object_stream | Object transform (rename/reshape) |
+| \`map { key: expr }\` | object_stream | object_stream | Expression transform (\`score: score * 2\`) |
+| \`save /path\` | object_stream | none | Write records to path (terminal) |
+| \`publish /topic\` | object_stream | none | Publish records to topic (terminal) |
+| \`tee /path\` | object_stream | object_stream | Side-write copy, stream continues |
+| \`count\` | object_stream | object_stream | Produce \`{ count: N }\` → schema: \`{ count: number }\` |
+| \`group-by field\` | object_stream | object_stream | Produce \`[{ key, items }]\` → schema: \`{ key: T, items: T[] }\` |
+| \`output "msg"\` | object_stream | object_stream | Emit text message, pass-through |
+| \`output expr\` | object_stream | object_stream | Evaluate expression per record, emit as text, pass-through |
+| \`input "prompt"\` | none | object_stream | Prompt for input → schema: \`{ prompt: string, response: string }\` |
+| \`fanout { branch1, branch2 }\` | object_stream | object_stream | Split stream to parallel branches |
+| \`action /path\` | object_stream | object_stream | Execute external action via \`world.exec\` |
+| \`route field { "v1" -> job j1, ... }\` | object_stream | none | Dispatch items by field value to named jobs |
+| \`lookup /path on key\` | object_stream | object_stream | Left-join with data at path on key field |
+
+**Type rule**: each stage's output type must match the next stage's input type. \`save\`, \`publish\`, and \`route\` output \`none\` — nothing can follow them in a pipeline.
+
+## Params & Variables
+
+\`\`\`ash
+param min_score = 70
+let bonus = 10
+let total = find /data/users | count
+job q { find /data/users | where score > $min_score | map { adjusted: score * 2 + $bonus } | save /data/out }
+\`\`\`
+
+\`param\` declares a named default that callers can override at execution time.
+\`let\` binds a name to a literal or a pipeline result. Reference with \`$name\` in where clauses and expressions.
+
+## Expressions
+
+Arithmetic: \`score * 2 + $bonus\`, \`price - discount\`
+String concat: \`name + " " + surname\`
+Usable in \`map { key: expr }\` and runtime \`let\`.
+
+## Triggers
+
+\`\`\`ash
+job ingest on /data/raw:created { find /data/raw | save /data/clean }
+job ticker on cron("*/5 * * * *") { find /metrics | save /snapshots }
+\`\`\`
+
+\`on /path:event\` — runs when the named event fires on the given path.
+\`on cron("expr")\` — runs on a cron schedule. Standard 5-field cron expression (minute hour day-of-month month day-of-week).
+
+## Annotations
+
+Annotations decorate jobs with metadata. Place them on lines before \`job\`:
+
+| Annotation | Description |
+|------------|-------------|
+| \`@approval(human)\` | Require human approval before execution |
+| \`@approval(auto)\` | Auto-approved |
+| \`@retry(N)\` | Retry up to N times on failure |
+| \`@timeout(ms)\` | Timeout in milliseconds |
+| \`@readonly\` | Job must not contain \`save\` or \`publish\` |
+| \`@on_error(strategy)\` | Error handling: \`skip\` (continue), \`save /path\` (write errors), \`fail\` (default) |
+| \`@caps(op /path, ...)\` | Capability declaration: restricts which paths the job can access |
+| \`@budget(dim limit, ...)\` | Runtime resource limits per job execution |
+
+\`\`\`ash
+@caps(read /data/*, write /out/*, exec /api/enrich)
+@on_error(skip)
+@retry(3)
+job resilient { find /data/raw | action /api/enrich | save /out/enriched }
+\`\`\`
+
+\`@caps\` operations: \`read\` (find, lookup), \`write\` (save, publish, tee), \`exec\` (action).
+Paths support \`*\` wildcard: \`/data/*\` matches \`/data/users\`, \`/data/orders/active\`.
+Compiler statically verifies all stage paths are within declared caps.
+
+\`@budget\` dimensions: \`actions\` (exec calls), \`writes\` (save/publish/tee), \`records\` (stream items), \`tokens\` (LLM consumption), \`cost\` (dollar cost).
+Runtime throws when any dimension exceeds its limit. Combine with \`@on_error(skip)\` for soft enforcement.
+
+\`\`\`ash
+@budget(actions 50, records 10000)
+@caps(read /data/*, write /out/*, exec /api/enrich)
+job bounded { find /data/raw | action /api/enrich | save /out/enriched }
+\`\`\`
+
+## Paths
+
+All paths start with \`/\`. Convention: \`/domain/collection\` (e.g. \`/world/users\`, \`/data/output\`).
+`;

package/src/template.test.ts

@@ -0,0 +1,210 @@
+import { describe, expect, it } from "vitest";
+import {
+  getNestedValue,
+  resolveTemplate,
+  resolveActionParams,
+  resolveTemplatePath,
+  hasTemplates,
+} from "./template";
+
+describe("getNestedValue", () => {
+  it("accesses top-level field", () => {
+    expect(getNestedValue({ name: "world" }, "name")).toBe("world");
+  });
+
+  it("accesses nested field via dot notation", () => {
+    expect(getNestedValue({ data: { id: 42 } }, "data.id")).toBe(42);
+  });
+
+  it("accesses array element by index", () => {
+    expect(getNestedValue({ items: [{ name: "first" }] }, "items.0.name")).toBe("first");
+  });
+
+  it("returns undefined for missing field", () => {
+    expect(getNestedValue({}, "missing")).toBeUndefined();
+  });
+
+  it("returns undefined for missing nested field", () => {
+    expect(getNestedValue({ data: {} }, "data.missing.deep")).toBeUndefined();
+  });
+
+  it("returns undefined when traversing through non-object", () => {
+    expect(getNestedValue({ x: "string" }, "x.length")).toBeUndefined();
+  });
+
+  it("returns undefined for null object", () => {
+    expect(getNestedValue(null, "x")).toBeUndefined();
+  });
+});
+
+describe("resolveTemplate", () => {
+  // Basic resolution
+  it("resolves simple field in string", () => {
+    expect(resolveTemplate("hello ${name}", { name: "world" })).toBe("hello world");
+  });
+
+  it("stringifies number in mixed context", () => {
+    expect(resolveTemplate("id=${data.id}", { data: { id: 42 } })).toBe("id=42");
+  });
+
+  it("resolves missing field to empty string", () => {
+    expect(resolveTemplate("${missing}", {})).toBe("");
+  });
+
+  it("resolves multiple templates", () => {
+    expect(resolveTemplate("${a} and ${b}", { a: "x", b: "y" })).toBe("x and y");
+  });
+
+  it("resolves array index access", () => {
+    expect(resolveTemplate("${items.0.name}", { items: [{ name: "first" }] })).toBe("first");
+  });
+
+  // Type preservation (whole-value)
+  it("preserves number type for whole-value", () => {
+    const result = resolveTemplate("${count}", { count: 42 });
+    expect(result).toBe(42);
+    expect(typeof result).toBe("number");
+  });
+
+  it("preserves object type for whole-value", () => {
+    const obj = { a: 1 };
+    expect(resolveTemplate("${obj}", { obj })).toEqual({ a: 1 });
+  });
+
+  it("preserves array type for whole-value", () => {
+    expect(resolveTemplate("${arr}", { arr: [1, 2, 3] })).toEqual([1, 2, 3]);
+  });
+
+  it("preserves boolean type for whole-value", () => {
+    const result = resolveTemplate("${flag}", { flag: true });
+    expect(result).toBe(true);
+    expect(typeof result).toBe("boolean");
+  });
+
+  // Mixed context stringify
+  it("JSON.stringifies object in mixed context", () => {
+    expect(resolveTemplate("val=${obj}", { obj: { a: 1 } })).toBe('val={"a":1}');
+  });
+
+  it("JSON.stringifies array in mixed context", () => {
+    expect(resolveTemplate("arr=${arr}", { arr: [1, 2] })).toBe("arr=[1,2]");
+  });
+
+  // Escape handling
+  it("preserves escaped template as literal", () => {
+    // After lexer processing, \$ becomes \\$ in token value
+    expect(resolveTemplate("\\${literal}", { literal: "x" })).toBe("${literal}");
+  });
+
+  it("resolves non-escaped but preserves escaped in same string", () => {
+    expect(resolveTemplate("${name} and \\${literal}", { name: "Bob", literal: "x" })).toBe(
+      "Bob and ${literal}",
+    );
+  });
+
+  // Non-string passthrough
+  it("returns non-string values as-is", () => {
+    expect(resolveTemplate(42, {})).toBe(42);
+    expect(resolveTemplate(true, {})).toBe(true);
+    expect(resolveTemplate(null, {})).toBeNull();
+  });
+
+  // Null/undefined in resolved values
+  it("resolves null field to empty string", () => {
+    expect(resolveTemplate("${x}", { x: null })).toBe("");
+  });
+
+  it("resolves null in mixed to empty string", () => {
+    expect(resolveTemplate("val=${x}", { x: null })).toBe("val=");
+  });
+
+  // Size limit
+  it("throws on resolved value exceeding 64KB", () => {
+    const bigString = "x".repeat(65537);
+    expect(() => resolveTemplate("${big}", { big: bigString })).toThrow("64KB");
+  });
+
+  it("allows resolved value at exactly 64KB", () => {
+    const okString = "x".repeat(65536);
+    expect(resolveTemplate("${ok}", { ok: okString })).toBe(okString);
+  });
+});
+
+describe("resolveActionParams", () => {
+  it("resolves all template params", () => {
+    const params = { text: "Hello ${name}", count: "${n}" };
+    const record = { name: "Bob", n: 5 };
+    const result = resolveActionParams(params, record);
+    expect(result).toEqual({ text: "Hello Bob", count: 5 });
+  });
+
+  it("passes through non-string values", () => {
+    const params = { limit: 10, flag: true };
+    const result = resolveActionParams(params, {});
+    expect(result).toEqual({ limit: 10, flag: true });
+  });
+
+  it("handles empty params", () => {
+    expect(resolveActionParams({}, {})).toEqual({});
+  });
+});
+
+describe("resolveTemplatePath", () => {
+  it("resolves template in path", () => {
+    expect(resolveTemplatePath("/msgs/${id}", { id: "123" })).toBe("/msgs/123");
+  });
+
+  it("resolves nested field in path", () => {
+    expect(resolveTemplatePath("/conv/${data.convId}/msgs", { data: { convId: "42" } })).toBe(
+      "/conv/42/msgs",
+    );
+  });
+
+  it("resolves multiple templates in path", () => {
+    expect(
+      resolveTemplatePath("/store/${cat}/.actions/${op}", { cat: "news", op: "save" }),
+    ).toBe("/store/news/.actions/save");
+  });
+
+  it("throws on empty segment from empty field", () => {
+    expect(() => resolveTemplatePath("/${empty}/.actions/run", { empty: "" })).toThrow(
+      "empty segment",
+    );
+  });
+
+  it("throws on .. in resolved path", () => {
+    expect(() => resolveTemplatePath("/base/${field}/run", { field: ".." })).toThrow(
+      "path traversal",
+    );
+  });
+
+  it("returns static path unchanged", () => {
+    expect(resolveTemplatePath("/static/path", {})).toBe("/static/path");
+  });
+
+  it("resolves number field to string in path", () => {
+    expect(resolveTemplatePath("/items/${id}", { id: 42 })).toBe("/items/42");
+  });
+});
+
+describe("hasTemplates", () => {
+  it("detects template in string", () => {
+    expect(hasTemplates("hello ${name}")).toBe(true);
+  });
+
+  it("returns false for no template", () => {
+    expect(hasTemplates("hello world")).toBe(false);
+  });
+
+  it("returns false for escaped template", () => {
+    expect(hasTemplates("\\${literal}")).toBe(false);
+  });
+
+  it("returns false for non-string", () => {
+    expect(hasTemplates(42)).toBe(false);
+  });
+
+  it("detects template among escaped", () => {
+    expect(hasTemplates("\\${esc} and ${real}")).toBe(true);
+  });
+});

package/src/template.ts

@@ -0,0 +1,139 @@
+/**
+ * ASH Template Parameter Resolution
+ *
+ * Resolves ${field} references in action parameters and paths
+ * against the current stream record.
+ */
+
+const MAX_RESOLVED_SIZE = 65536; // 64KB
+
+// Match ${identifier.path} but not \${...}
+const TEMPLATE_RE = /(?<!\\)\$\{([^}]+)\}/g;
+// Match whole-value template: entire string is a single ${field}
+const WHOLE_VALUE_RE = /^\$\{([^}]+)\}$/;
+
+/**
+ * Access a nested value via dot-separated path.
+ * Supports object fields and array indices (numeric keys).
+ *
+ * getNestedValue({data: {id: 42}}, "data.id") → 42
+ * getNestedValue({items: [{name: "a"}]}, "items.0.name") → "a"
+ */
+export function getNestedValue(obj: unknown, path: string): unknown {
+  const keys = path.split(".");
+  let current: unknown = obj;
+  for (const key of keys) {
+    if (current === null || current === undefined) return undefined;
+    if (typeof current !== "object") return undefined;
+    current = (current as Record<string, unknown>)[key];
+  }
+  return current;
+}
+
+/**
+ * Resolve a single template value against a stream record.
+ *
+ * - Non-string values pass through unchanged
+ * - Whole-value "${field}" preserves the field's type
+ * - Mixed "text ${field}" stringifies substitutions (JSON.stringify for objects)
+ * - Escaped \${...} produces literal ${...}
+ * - Missing fields resolve to ""
+ * - Resolved strings > 64KB throw
+ */
+export function resolveTemplate(value: unknown, record: Record<string, unknown>): unknown {
+  if (typeof value !== "string") return value;
+
+  // Whole-value: single ${field} with nothing else → preserve type
+  const wholeMatch = value.match(WHOLE_VALUE_RE);
+  if (wholeMatch) {
+    const resolved = getNestedValue(record, wholeMatch[1]);
+    if (resolved === undefined || resolved === null) return "";
+    // Size check for string values
+    if (typeof resolved === "string" && resolved.length > MAX_RESOLVED_SIZE) {
+      throw new Error(
+        `Template resolution error: resolved value exceeds 64KB limit (${resolved.length} bytes)`,
+      );
+    }
+    return resolved;
+  }
+
+  // Mixed template: replace ${...} with stringified values
+  const result = value.replace(TEMPLATE_RE, (_, field: string) => {
+    const val = getNestedValue(record, field);
+    if (val === undefined || val === null) return "";
+    if (typeof val === "object") return JSON.stringify(val);
+    return String(val);
+  });
+
+  // Strip escape: \${ → ${
+  const final = result.replace(/\\\$/g, "$");
+
+  // Size check
+  if (final.length > MAX_RESOLVED_SIZE) {
+    throw new Error(
+      `Template resolution error: resolved value exceeds 64KB limit (${final.length} bytes)`,
+    );
+  }
+
+  return final;
+}
+
+/**
+ * Resolve all template references in an action's parameters.
+ */
+export function resolveActionParams(
+  params: Record<string, unknown>,
+  record: Record<string, unknown>,
+): Record<string, unknown> {
+  const resolved: Record<string, unknown> = {};
+  for (const [key, value] of Object.entries(params)) {
+    resolved[key] = resolveTemplate(value, record);
+  }
+  return resolved;
+}
+
+/**
+ * Resolve template references in a path string.
+ * Always returns a string. Validates the resolved path:
+ * - No empty segments (//)
+ * - No path traversal (..)
+ */
+export function resolveTemplatePath(
+  path: string,
+  record: Record<string, unknown>,
+): string {
+  if (!path.includes("${")) return path;
+
+  const resolved = path.replace(TEMPLATE_RE, (_, field: string) => {
+    const val = getNestedValue(record, field);
+    if (val === undefined || val === null) return "";
+    return String(val);
+  });
+
+  // Strip escape: \${ → ${
+  const final = resolved.replace(/\\\$/g, "$");
+
+  // Validate resolved path
+  if (final.includes("//")) {
+    throw new Error(
+      `Template path resolution error: resolved path '${final}' contains empty segment (//). Check that template fields are not empty.`,
+    );
+  }
+  if (final.split("/").some((seg) => seg === "..")) {
+    throw new Error(
+      `Template path resolution error: resolved path '${final}' contains path traversal (..)`,
+    );
+  }
+
+  return final;
+}
+
+/**
+ * Check if a value contains any unescaped ${...} template references.
+ */
+export function hasTemplates(value: unknown): boolean {
+  if (typeof value !== "string") return false;
+  // Reset regex lastIndex since it's global
+  TEMPLATE_RE.lastIndex = 0;
+  return TEMPLATE_RE.test(value);
+}