@rigkit/engine 0.2.3 → 0.2.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rigkit/engine",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "type": "module",
   "repository": {
     "type": "git",

package/src/authoring.ts

@@ -25,7 +25,7 @@ import type {
   WorkflowWorkspaceDefinition,
 } from "./types.ts";
 
-const reservedTaskContextKeys = new Set(["ctx", "runtime", "providers"]);
+const reservedTaskContextKeys = new Set(["ctx", "runtime", "providers", "step"]);
 const reservedHostOperationIds = new Set<string>(RESERVED_WORKFLOW_OPERATION_IDS);
 
 const readEnv = (name: string, fallback?: string): string => {
@@ -51,9 +51,17 @@ export function workflow<const Name extends string, const Providers extends Work
     providers: options.providers,
     sequence: <InputContext extends JsonObject = {}>(sequenceName: string) =>
       createSequence(app as unknown as WorkflowDefinition<string, Providers>, sequenceName, []),
-    task: (taskName: string, optionsOrHandler: WorkflowTaskOptions | WorkflowTaskHandler<Providers, {}, any>, maybeHandler?: WorkflowTaskHandler<Providers, {}, any>) =>
+    task: (
+      taskName: string,
+      optionsOrHandler: WorkflowTaskOptions | WorkflowTaskHandler<Providers, {}, never, any>,
+      maybeHandler?: WorkflowTaskHandler<Providers, {}, never, any>,
+    ) =>
       createTask(app as unknown as WorkflowDefinition<string, Providers>, taskName, optionsOrHandler as any, maybeHandler as any),
-    step: (taskName: string, optionsOrHandler: WorkflowTaskOptions | WorkflowTaskHandler<Providers, {}, any>, maybeHandler?: WorkflowTaskHandler<Providers, {}, any>) =>
+    step: (
+      taskName: string,
+      optionsOrHandler: WorkflowTaskOptions | WorkflowTaskHandler<Providers, {}, never, any>,
+      maybeHandler?: WorkflowTaskHandler<Providers, {}, never, any>,
+    ) =>
       createTask(app as unknown as WorkflowDefinition<string, Providers>, taskName, optionsOrHandler as any, maybeHandler as any),
   };
 
@@ -127,6 +135,7 @@ function createSequence<
   WorkspaceData extends object = JsonObject,
   OperationIds extends string = never,
   WorkspaceOperationIds extends string = never,
+  PreviousTaskIds extends string = never,
 >(
   app: WorkflowDefinition<string, Providers>,
   name: string,
@@ -140,7 +149,8 @@ function createSequence<
   OutputContext,
   WorkspaceData,
   OperationIds,
-  WorkspaceOperationIds
+  WorkspaceOperationIds,
+  PreviousTaskIds
 > {
   const node = {
     kind: "rigkit.workflow-node" as const,
@@ -153,16 +163,16 @@ function createSequence<
     workspaceOperations,
     task: (
       taskName: string,
-      optionsOrHandler: WorkflowTaskOptions | WorkflowTaskHandler<Providers, OutputContext, any>,
-      maybeHandler?: WorkflowTaskHandler<Providers, OutputContext, any>,
+      optionsOrHandler: WorkflowTaskOptions | WorkflowTaskHandler<Providers, OutputContext, PreviousTaskIds, any>,
+      maybeHandler?: WorkflowTaskHandler<Providers, OutputContext, PreviousTaskIds, any>,
     ) => {
       const task = createTask(app, taskName, optionsOrHandler as any, maybeHandler as any);
       return createSequence(app, name, [...children, task], workspace, operations, workspaceOperations);
     },
     step: (
       taskName: string,
-      optionsOrHandler: WorkflowTaskOptions | WorkflowTaskHandler<Providers, OutputContext, any>,
-      maybeHandler?: WorkflowTaskHandler<Providers, OutputContext, any>,
+      optionsOrHandler: WorkflowTaskOptions | WorkflowTaskHandler<Providers, OutputContext, PreviousTaskIds, any>,
+      maybeHandler?: WorkflowTaskHandler<Providers, OutputContext, PreviousTaskIds, any>,
     ) => {
       const task = createTask(app, taskName, optionsOrHandler as any, maybeHandler as any);
       return createSequence(app, name, [...children, task], workspace, operations, workspaceOperations);
@@ -209,7 +219,8 @@ function createSequence<
     OutputContext,
     WorkspaceData,
     OperationIds,
-    WorkspaceOperationIds
+    WorkspaceOperationIds,
+    PreviousTaskIds
   >;
 }
 
@@ -319,15 +330,19 @@ function createOperationInputBuilder<Input extends object>(
   } as WorkflowOperationInputBuilder<Input>;
 }
 
-function createTask<Providers extends WorkflowProviderMap, InputContext extends JsonObject>(
+function createTask<
+  Providers extends WorkflowProviderMap,
+  InputContext extends JsonObject,
+  PreviousTaskIds extends string = string,
+>(
   app: WorkflowDefinition<string, Providers>,
   name: string,
-  optionsOrHandler: WorkflowTaskOptions | WorkflowTaskHandler<Providers, InputContext, any>,
-  maybeHandler?: WorkflowTaskHandler<Providers, InputContext, any>,
+  optionsOrHandler: WorkflowTaskOptions | WorkflowTaskHandler<Providers, InputContext, PreviousTaskIds, any>,
+  maybeHandler?: WorkflowTaskHandler<Providers, InputContext, PreviousTaskIds, any>,
 ): WorkflowTaskNode<Providers, InputContext, any> {
   const options = typeof optionsOrHandler === "function" ? undefined : optionsOrHandler;
   const handler = (typeof optionsOrHandler === "function" ? optionsOrHandler : maybeHandler) as
-    | WorkflowTaskHandler<Providers, InputContext, any>
+    | WorkflowTaskHandler<Providers, InputContext, PreviousTaskIds, any>
     | undefined;
   if (!handler) throw new Error(`Task ${name} is missing a handler`);
 

package/src/authoring.typecheck.ts

@@ -10,7 +10,7 @@ sequence("normal-operation-ids")
   });
 
 sequence("workspace-operation-ids")
-  .step("prepare", async () => ({ snapshotId: "snap-1" }))
+  .step("prepare", async () => ({ ctx: { snapshotId: "snap-1" } }))
   .workspace({
     create: async ({ workflow, workspace }) => {
       const snapshotId: string = workflow.ctx.snapshotId;
@@ -44,6 +44,26 @@ sequence("workspace-operation-ids")
     run: async () => null,
   });
 
+sequence("typed-step-invalidation")
+  .step("github-auth", async () => ({ ctx: { token: "ok" } }))
+  .step("check-auth", async ({ step }) => {
+    const token: string = step.ctx.token;
+    void token;
+    return step.invalidate("github-auth");
+  });
+
+sequence("typed-step-invalidation-targets")
+  .step("github-auth", async () => ({ ctx: { token: "ok" } }))
+  .step("check-auth", async ({ step }) => {
+    // @ts-expect-error invalidation target must be a previous task id
+    return step.invalidate("missing-auth");
+  });
+
+sequence("duplicate-step-id")
+  .step("prepare" as const, async () => ({ ctx: { snapshotId: "snap-1" } }))
+  // @ts-expect-error duplicate task ids are rejected for literal ids
+  .step("prepare" as const, async ({ step }) => ({ ctx: step.ctx }));
+
 sequence("reserved-operation-id")
   // @ts-expect-error reserved operation ids are rejected for literal ids
   .operation("create" as const, {

package/src/engine.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, test } from "bun:test";
 import { Database } from "bun:sqlite";
-import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
+import { mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { createDevMachineEngine, type InteractionPresentationRequest } from "./engine.ts";
@@ -27,34 +27,34 @@ describe("DevMachineEngine workflow runtime", () => {
           },
         });
 
-        const base = app.sequence("base").task("first", async ({ runtime, test }) => {
-          runtime.log("preparing base\\n", { label: "setup" });
+        const base = app.sequence("base").task("first", async ({ step, test }) => {
+          step.log("preparing base\\n", { label: "setup" });
           const vm = await test.createVm();
           await vm.exec("touch /tmp/first", { name: "touch first" });
           if (!(await vm.exists("/tmp/first"))) throw new Error("first was not created");
-          return { first: true, vm: await vm.snapshotRef() };
+          return { ctx: { first: true, vm: await vm.snapshotRef() } };
         });
 
-        const left = app.sequence("left").task("second", async ({ ctx, test }) => {
-          if (!ctx.first) throw new Error("missing first context");
-          const vm = await test.fromSnapshot(ctx.vm);
+        const left = app.sequence("left").task("second", async ({ step, test }) => {
+          if (!step.ctx.first) throw new Error("missing first context");
+          const vm = await test.fromSnapshot(step.ctx.vm);
           await vm.exec("touch /tmp/second", { name: "touch second" });
-          return { second: true, vm: await vm.snapshotRef() };
+          return { ctx: { second: true, vm: await vm.snapshotRef() } };
         });
 
-        const right = app.sequence("right").task("data", async ({ ctx }) => {
-          if (!ctx.first) throw new Error("missing first context");
-          return { data: "right-ready" };
+        const right = app.sequence("right").task("data", async ({ step }) => {
+          if (!step.ctx.first) throw new Error("missing first context");
+          return { ctx: { data: "right-ready" } };
         });
 
         export default app
           .sequence("root")
           .add(base)
           .parallel({ left, right })
-          .task("join", async ({ ctx }) => {
-            if (!ctx.left.second) throw new Error("missing left context");
-            if (ctx.right.data !== "right-ready") throw new Error("missing right context");
-            return { vm: ctx.left.vm, summary: ctx.right.data };
+          .task("join", async ({ step }) => {
+            if (!step.ctx.left.second) throw new Error("missing left context");
+            if (step.ctx.right.data !== "right-ready") throw new Error("missing right context");
+            return { ctx: { vm: step.ctx.left.vm, summary: step.ctx.right.data } };
           })
           .workspace({
             create: async ({ providers, workflow, workspace, local }) => {
@@ -189,8 +189,10 @@ describe("DevMachineEngine workflow runtime", () => {
             const vm = await providers.test.createVm();
             await vm.exec("touch /tmp/template", { name: "prepare template" });
             return {
-              vm: await vm.snapshotRef(),
-              repoPath: "/workspace/repo",
+              ctx: {
+                vm: await vm.snapshotRef(),
+                repoPath: "/workspace/repo",
+              },
             };
           })
           .workspace({
@@ -279,8 +281,8 @@ describe("DevMachineEngine workflow runtime", () => {
       `
         import { defineConfig, sequence } from "${import.meta.dir}/index.ts";
 
-        const api = sequence("api").step("ready", async () => ({ api: true }));
-        const web = sequence("web").step("ready", async () => ({ web: true }));
+        const api = sequence("api").step("ready", async () => ({ ctx: { api: true } }));
+        const web = sequence("web").step("ready", async () => ({ ctx: { web: true } }));
 
         export default defineConfig({
           providers: {},
@@ -310,7 +312,7 @@ describe("DevMachineEngine workflow runtime", () => {
       `
         import { defineConfig, sequence } from "${import.meta.dir}/index.ts";
 
-        const root = sequence("factory-test").step("ready", async () => ({ ready: true }));
+        const root = sequence("factory-test").step("ready", async () => ({ ctx: { ready: true } }));
 
         export default defineConfig({
           providers: {},
@@ -432,7 +434,7 @@ describe("DevMachineEngine workflow runtime", () => {
           const app = workflow("handler-cache", { providers: {} });
 
           export default app.sequence("root").task("value", async () => {
-            return { value: "${value}" };
+            return { ctx: { value: "${value}" } };
           });
         `,
       );
@@ -490,7 +492,7 @@ describe("DevMachineEngine workflow runtime", () => {
           },
         });
 
-        export default app.sequence("root").task("ready", async () => ({ ready: true }));
+        export default app.sequence("root").task("ready", async () => ({ ctx: { ready: true } }));
       `,
     );
 
@@ -533,6 +535,73 @@ describe("DevMachineEngine workflow runtime", () => {
     expect(metadata["config.path"]).toBe(join(projectDir, "rig.config.ts"));
   });
 
+  test("stores provider host JSON state outside project state", async () => {
+    const projectDir = mkdtempSync(join(tmpdir(), "rigkit-"));
+    const hostStorageDir = join(projectDir, ".host-storage");
+    const opened: string[] = [];
+    const plugin: BaseProviderPlugin = {
+      providerId: "test",
+      async createProvider({ storage, hostStorage, local }) {
+        storage.set("project", { value: "state" });
+        hostStorage.set("token", { value: "secret" });
+        await local.open("rigkit://provider-auth");
+        return new FakeWorkflowProvider();
+      },
+    };
+
+    writeFileSync(
+      join(projectDir, "rig.config.ts"),
+      `
+        import { defineProvider, workflow } from "${import.meta.dir}/index.ts";
+
+        const app = workflow("provider-host-storage", {
+          providers: {
+            test: defineProvider("test", {}),
+          },
+        });
+
+        export default app.sequence("root").task("ready", async () => ({ ctx: { ready: true } }));
+      `,
+    );
+
+    const engine = await createDevMachineEngine({
+      projectDir,
+      hostStorageDir,
+      providers: [plugin],
+      local: {
+        open: async (target) => {
+          opened.push(target);
+        },
+      },
+    });
+
+    await engine.load();
+    await engine.plan();
+
+    expect(opened).toEqual(["rigkit://provider-auth"]);
+
+    const files = readdirSync(hostStorageDir);
+    expect(files).toHaveLength(1);
+    const hostState = JSON.parse(readFileSync(join(hostStorageDir, files[0]!), "utf8"));
+    expect(hostState.records.token.value.value).toBe("secret");
+
+    const main = new Database(engine.getProjectInfo().statePath);
+    const projectRow = main
+      .query<{ value_json: string }, []>(
+        "select value_json from provider_state where provider_id = 'test' and key = 'project'",
+      )
+      .get();
+    const leakedHostRow = main
+      .query<{ value_json: string }, []>(
+        "select value_json from provider_state where provider_id = 'test' and key = 'token'",
+      )
+      .get();
+    main.close();
+
+    expect(projectRow ? JSON.parse(projectRow.value_json).value : undefined).toBe("state");
+    expect(leakedHostRow).toBeNull();
+  });
+
   test("rejects task outputs that are not JSON serializable", async () => {
     const projectDir = mkdtempSync(join(tmpdir(), "rigkit-"));
     writeFileSync(
@@ -547,7 +616,7 @@ describe("DevMachineEngine workflow runtime", () => {
         });
 
         export default app.sequence("bad").task("returns-function", async () => {
-          return { fn: () => "nope" };
+          return { ctx: { fn: () => "nope" } };
         });
       `,
     );
@@ -576,7 +645,7 @@ describe("DevMachineEngine workflow runtime", () => {
 
         export default app.sequence("auth").task("login", async ({ test }) => {
           const result = await test.openTerminal("GitHub auth", "gh auth login");
-          return { finished: result.finished };
+          return { ctx: { finished: result.finished } };
         });
       `,
     );
@@ -622,7 +691,7 @@ describe("DevMachineEngine workflow runtime", () => {
 
         export default app.sequence("auth").task("login", async ({ test }) => {
           const result = await test.openTerminal("GitHub auth", "gh auth login");
-          return { finished: result.finished };
+          return { ctx: { finished: result.finished } };
         });
       `,
     );
@@ -676,7 +745,7 @@ describe("DevMachineEngine workflow runtime", () => {
         export default app.sequence("setup").task("touch", async ({ test }) => {
           const vm = await test.createVm();
           await vm.exec("touch /tmp/setup", { name: "touch setup" });
-          return { vm: await vm.snapshotRef() };
+          return { ctx: { vm: await vm.snapshotRef() } };
         });
       `,
     );
@@ -734,8 +803,8 @@ describe("DevMachineEngine workflow runtime", () => {
 
         export default app.sequence("schema").task("value", { output: schema }, async () => {
           return process.env.RIGKIT_SCHEMA_MODE === "next"
-            ? { value: "ok", next: true }
-            : { value: "ok" };
+            ? { ctx: { value: "ok", next: true } }
+            : { ctx: { value: "ok" } };
         });
       `,
     );
@@ -767,6 +836,94 @@ describe("DevMachineEngine workflow runtime", () => {
       }
     }
   });
+
+  test("expires task cache when cacheTTL has elapsed", async () => {
+    const projectDir = mkdtempSync(join(tmpdir(), "rigkit-cache-ttl-"));
+    writeFileSync(
+      join(projectDir, "rig.config.ts"),
+      `
+        import { sequence } from "${import.meta.dir}/index.ts";
+
+        export default sequence("ttl").task("daily-check", { cacheTTL: "1d" }, async () => {
+          return { ctx: { checked: true } };
+        });
+      `,
+    );
+
+    const engine = await createDevMachineEngine({ projectDir });
+    await engine.load();
+    await engine.apply();
+    expect((await engine.plan()).cachedNodeCount).toBe(1);
+
+    const db = new Database(engine.getProjectInfo().statePath);
+    db.run("update workflow_node_runs set created_at = ?", [
+      new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),
+    ]);
+    db.close();
+
+    const expired = await engine.plan();
+    expect(expired.cachedNodeCount).toBe(0);
+    expect(expired.nodes[0]?.status).toBe("pending");
+  });
+
+  test("step.invalidate invalidates a previous task and replays from that point", async () => {
+    const projectDir = mkdtempSync(join(tmpdir(), "rigkit-invalidate-"));
+    const previous = {
+      authCount: process.env.RIGKIT_AUTH_COUNT,
+      checkCount: process.env.RIGKIT_CHECK_COUNT,
+      forceReauth: process.env.RIGKIT_FORCE_REAUTH,
+    };
+    process.env.RIGKIT_AUTH_COUNT = "0";
+    process.env.RIGKIT_CHECK_COUNT = "0";
+    process.env.RIGKIT_FORCE_REAUTH = "0";
+
+    writeFileSync(
+      join(projectDir, "rig.config.ts"),
+      `
+        import { sequence } from "${import.meta.dir}/index.ts";
+
+        export default sequence("reauth")
+          .task("prepare", async () => ({ ctx: { prepared: true } }))
+          .task("github-auth", async () => {
+            const count = Number(process.env.RIGKIT_AUTH_COUNT ?? "0") + 1;
+            process.env.RIGKIT_AUTH_COUNT = String(count);
+            return { ctx: { token: "token-" + count } };
+          })
+          .task("check-auth", { cacheTTL: 0 }, async ({ step }) => {
+            const count = Number(process.env.RIGKIT_CHECK_COUNT ?? "0") + 1;
+            process.env.RIGKIT_CHECK_COUNT = String(count);
+            if (process.env.RIGKIT_FORCE_REAUTH === "1") {
+              process.env.RIGKIT_FORCE_REAUTH = "0";
+              return step.invalidate("github-auth");
+            }
+            return { ctx: step.ctx };
+          });
+      `,
+    );
+
+    try {
+      const engine = await createDevMachineEngine({ projectDir });
+      await engine.load();
+
+      const first = await engine.apply();
+      expect(first.context.token).toBe("token-1");
+      expect(process.env.RIGKIT_AUTH_COUNT).toBe("1");
+      expect(process.env.RIGKIT_CHECK_COUNT).toBe("1");
+
+      process.env.RIGKIT_FORCE_REAUTH = "1";
+      const second = await engine.apply();
+      expect(second.context.token).toBe("token-2");
+      expect(process.env.RIGKIT_AUTH_COUNT).toBe("2");
+      expect(process.env.RIGKIT_CHECK_COUNT).toBe("3");
+
+      const validRuns = engine.listNodeRuns().filter((run) => !run.invalidated);
+      expect(validRuns.map((run) => run.nodePath).sort()).toEqual(["check-auth", "github-auth", "prepare"]);
+    } finally {
+      restoreEnv("RIGKIT_AUTH_COUNT", previous.authCount);
+      restoreEnv("RIGKIT_CHECK_COUNT", previous.checkCount);
+      restoreEnv("RIGKIT_FORCE_REAUTH", previous.forceReauth);
+    }
+  });
 });
 
 type FakeSnapshotRef = {
@@ -879,6 +1036,14 @@ function result(ok: boolean): ExecResult {
   return { stdout: "", stderr: "", exitCode: ok ? 0 : 1, ok };
 }
 
+function restoreEnv(name: string, value: string | undefined): void {
+  if (value === undefined) {
+    delete process.env[name];
+  } else {
+    process.env[name] = value;
+  }
+}
+
 function isFakeSnapshotRef(value: unknown): value is FakeSnapshotRef {
   return Boolean(
     value &&