@rigkit/engine 0.1.8

package/README.md

@@ -0,0 +1,5 @@
+# @rigkit/engine
+
+Runtime engine for Rigkit workflows.
+
+This package loads `rig.config.ts`, evaluates workflow nodes, manages graph-based `.rigkit/state.sqlite` node-run cache, runs workspace lifecycle hooks, talks to registered workflow providers, presents provider-owned interaction URLs, and exposes APIs used by the CLI and future app.

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@rigkit/engine",
+  "version": "0.1.8",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/freestyle-sh/rigkit.git",
+    "directory": "packages/engine"
+  },
+  "exports": {
+    ".": "./src/index.ts",
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "src",
+    "README.md"
+  ],
+  "dependencies": {
+    "drizzle-kit": "^0.31.10",
+    "drizzle-orm": "^0.45.2"
+  },
+  "devDependencies": {
+    "@types/bun": "latest",
+    "typescript": "latest"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsc --noEmit",
+    "typecheck": "tsc --noEmit",
+    "test": "bun test"
+  }
+}

package/src/authoring.ts

@@ -0,0 +1,334 @@
+import type {
+  EnvResolver,
+  RigkitConfigDefinition,
+  JsonObject,
+  OutputSchema,
+  OutputSchemaValue,
+  WorkflowDefinition,
+  WorkflowCreateDefinition,
+  WorkflowCreateHandler,
+  WorkflowInputFieldDefinition,
+  WorkflowInputShape,
+  WorkflowOperationDefinition,
+  WorkflowOperationInputBuilder,
+  WorkflowOperationInputHelpers,
+  WorkflowOperationOptions,
+  WorkflowNodeDefinition,
+  WorkflowProviderDefinition,
+  WorkflowProviderMap,
+  WorkflowSequenceBuilder,
+  WorkflowTaskHandler,
+  WorkflowTaskNode,
+  WorkflowTaskOptions,
+  WorkflowTaskResult,
+  WorkflowWorkspaceDefinition,
+} from "./types.ts";
+
+const reservedTaskContextKeys = new Set(["ctx", "runtime", "providers"]);
+const reservedHostOperationIds = new Set(["init", "doctor", "projects", "run", "ls", "help", "version", "completion"]);
+
+const readEnv = (name: string, fallback?: string): string => {
+  const value = process.env[name];
+  if (value !== undefined && value !== "") return value;
+  if (fallback !== undefined) return fallback;
+  throw new Error(`Missing required environment variable ${name}`);
+};
+
+export const env: EnvResolver = Object.assign(readEnv, {
+  secret: readEnv,
+});
+
+export function workflow<const Name extends string, const Providers extends WorkflowProviderMap>(
+  name: Name,
+  options: { providers: Providers },
+): WorkflowDefinition<Name, Providers> {
+  validateProviders(options.providers);
+
+  const app = {
+    kind: "rigkit.workflow" as const,
+    name,
+    providers: options.providers,
+    sequence: <InputContext extends JsonObject = {}>(sequenceName: string) =>
+      createSequence(app as unknown as WorkflowDefinition<string, Providers>, sequenceName, []),
+    task: (taskName: string, optionsOrHandler: WorkflowTaskOptions | WorkflowTaskHandler<Providers, {}, any>, maybeHandler?: WorkflowTaskHandler<Providers, {}, any>) =>
+      createTask(app as unknown as WorkflowDefinition<string, Providers>, taskName, optionsOrHandler as any, maybeHandler as any),
+    step: (taskName: string, optionsOrHandler: WorkflowTaskOptions | WorkflowTaskHandler<Providers, {}, any>, maybeHandler?: WorkflowTaskHandler<Providers, {}, any>) =>
+      createTask(app as unknown as WorkflowDefinition<string, Providers>, taskName, optionsOrHandler as any, maybeHandler as any),
+  };
+
+  return app as unknown as WorkflowDefinition<Name, Providers>;
+}
+
+export function sequence<
+  Providers extends WorkflowProviderMap = WorkflowProviderMap,
+  InputContext extends JsonObject = {},
+>(name: string): WorkflowSequenceBuilder<Providers, InputContext, InputContext> {
+  return workflow(name, { providers: {} as Providers }).sequence<InputContext>(name);
+}
+
+export function defineConfig<
+  const Providers extends WorkflowProviderMap,
+  const Workflows extends Record<string, WorkflowNodeDefinition<any, any, any>>,
+>(options: {
+  providers: Providers;
+  workflows: Workflows;
+}): RigkitConfigDefinition<Providers, Workflows> {
+  validateProviders(options.providers);
+  for (const [name, node] of Object.entries(options.workflows)) {
+    if (!isWorkflowNode(node)) {
+      throw new Error(`Workflow ${name} is not a valid Rigkit workflow node`);
+    }
+  }
+  return {
+    kind: "rigkit.config",
+    providers: options.providers,
+    workflows: options.workflows,
+  };
+}
+
+export function defineProvider<
+  const ProviderId extends string,
+  const Config extends object,
+  Runtime = unknown,
+  WorkspaceContext extends object = object,
+>(
+  providerId: ProviderId,
+  config: WorkflowProviderDefinition<ProviderId, Config, Runtime, WorkspaceContext>["config"],
+  plugin?: unknown,
+): WorkflowProviderDefinition<ProviderId, Config, Runtime, WorkspaceContext> {
+  return {
+    kind: "rigkit.provider",
+    providerId,
+    config,
+    plugin,
+  };
+}
+
+export function isWorkflow(value: unknown): value is WorkflowDefinition {
+  return Boolean(value && typeof value === "object" && getKind(value) === "rigkit.workflow");
+}
+
+export function isWorkflowNode(value: unknown): value is WorkflowNodeDefinition<any, any, any> {
+  return Boolean(value && typeof value === "object" && getKind(value) === "rigkit.workflow-node");
+}
+
+export function isRigkitConfig(value: unknown): value is RigkitConfigDefinition {
+  return Boolean(value && typeof value === "object" && getKind(value) === "rigkit.config");
+}
+
+export function isProviderDefinition(value: unknown): value is WorkflowProviderDefinition {
+  return Boolean(value && typeof value === "object" && getKind(value) === "rigkit.provider");
+}
+
+function createSequence<Providers extends WorkflowProviderMap, InputContext extends JsonObject, OutputContext extends JsonObject>(
+  app: WorkflowDefinition<string, Providers>,
+  name: string,
+  children: readonly WorkflowNodeDefinition<Providers, any, any>[],
+  workspace?: WorkflowWorkspaceDefinition<Providers, OutputContext>,
+  create?: WorkflowCreateDefinition<Providers, OutputContext>,
+  operations: readonly WorkflowOperationDefinition<Providers, any>[] = [],
+): WorkflowSequenceBuilder<Providers, InputContext, OutputContext> {
+  const node = {
+    kind: "rigkit.workflow-node" as const,
+    nodeKind: "sequence" as const,
+    name,
+    workflow: app,
+    children,
+    workspaceDefinition: workspace,
+    createDefinition: create,
+    operations,
+    task: (
+      taskName: string,
+      optionsOrHandler: WorkflowTaskOptions | WorkflowTaskHandler<Providers, OutputContext, any>,
+      maybeHandler?: WorkflowTaskHandler<Providers, OutputContext, any>,
+    ) => {
+      const task = createTask(app, taskName, optionsOrHandler as any, maybeHandler as any);
+      return createSequence(app, name, [...children, task], workspace, create, operations);
+    },
+    step: (
+      taskName: string,
+      optionsOrHandler: WorkflowTaskOptions | WorkflowTaskHandler<Providers, OutputContext, any>,
+      maybeHandler?: WorkflowTaskHandler<Providers, OutputContext, any>,
+    ) => {
+      const task = createTask(app, taskName, optionsOrHandler as any, maybeHandler as any);
+      return createSequence(app, name, [...children, task], workspace, create, operations);
+    },
+    add: (child: WorkflowNodeDefinition<Providers, any, any>) => {
+      assertSameWorkflow(app, child);
+      return createSequence(app, name, [...children, child], workspace, create, operations);
+    },
+    parallel: (branches: Record<string, WorkflowNodeDefinition<Providers, any, any>>) => {
+      for (const [branchName, branch] of Object.entries(branches)) {
+        assertSameWorkflow(app, branch);
+        if (!branchName) throw new Error(`Parallel branch names must be non-empty`);
+      }
+
+      const parallelNode: WorkflowNodeDefinition<Providers, OutputContext, any> & {
+        nodeKind: "parallel";
+        branches: Record<string, WorkflowNodeDefinition<Providers, any, any>>;
+      } = {
+        kind: "rigkit.workflow-node",
+        nodeKind: "parallel",
+        name: "parallel",
+        workflow: app,
+        branches,
+      };
+      return createSequence(app, name, [...children, parallelNode], workspace, create, operations);
+    },
+    workspace: (definition: WorkflowWorkspaceDefinition<Providers, OutputContext>) =>
+      createSequence(app, name, children, definition, create, operations),
+    create: (handler: WorkflowCreateHandler<Providers, OutputContext, any>) =>
+      createSequence(app, name, children, workspace, { handler }, operations),
+    operation: (id: string, options: WorkflowOperationOptions<Providers, any>) =>
+      createSequence(app, name, children, workspace, create, [...operations, createOperation(id, options)]),
+  };
+
+  return node as unknown as WorkflowSequenceBuilder<Providers, InputContext, OutputContext>;
+}
+
+function createOperation<Providers extends WorkflowProviderMap, Input extends object>(
+  id: string,
+  options: WorkflowOperationOptions<Providers, Input>,
+): WorkflowOperationDefinition<Providers, Input> {
+  const normalized = id.trim();
+  if (!normalized) throw new Error(`Operation ids must be non-empty`);
+  if (reservedHostOperationIds.has(normalized)) {
+    throw new Error(`Operation id ${normalized} is reserved by the Rigkit host`);
+  }
+  return {
+    id: normalized,
+    title: options.title,
+    description: options.description,
+    createsWorkspace: options.createsWorkspace,
+    requiredHostMethods: normalizeHostMethodRequirements(options.requiredHostMethods),
+    requiredHostCapabilities: normalizeHostCapabilityRequirements(options.requiredHostCapabilities),
+    input: typeof options.input === "function"
+      ? options.input(createOperationInputHelpers())
+      : options.input,
+    run: options.run,
+  };
+}
+
+function normalizeHostMethodRequirements(
+  methods: WorkflowOperationOptions<any, any>["requiredHostMethods"],
+) {
+  return methods?.map((method) => {
+    const id = method.id.trim();
+    if (!id) throw new Error(`Host method requirements must have non-empty ids`);
+    return {
+      id,
+      ...(method.modes?.length ? { modes: [...method.modes] } : {}),
+    };
+  });
+}
+
+function normalizeHostCapabilityRequirements(
+  capabilities: WorkflowOperationOptions<any, any>["requiredHostCapabilities"],
+) {
+  return capabilities?.map((capability) => {
+    const id = capability.id.trim();
+    if (!id) throw new Error(`Host capability requirements must have non-empty ids`);
+    const schemaHash = capability.schemaHash?.trim();
+    return {
+      id,
+      ...(schemaHash ? { schemaHash } : {}),
+    };
+  });
+}
+
+function createOperationInputHelpers(): WorkflowOperationInputHelpers {
+  return {
+    workspaceInput: (options) => createOperationInputBuilder([{
+      kind: "workspace",
+      name: options.name,
+      description: options.description,
+      position: options.position,
+      required: options.required ?? true,
+    }]),
+    string: (options) => ({
+      kind: "string",
+      name: options.name ?? "",
+      description: options.description,
+      position: options.position,
+      required: options.required ?? options.defaultValue === undefined,
+      defaultValue: options.defaultValue,
+    }),
+    boolean: (options) => ({
+      kind: "boolean",
+      name: options.name ?? "",
+      description: options.description,
+      position: options.position,
+      required: options.required ?? false,
+      defaultValue: options.defaultValue,
+    }),
+    number: (options) => ({
+      kind: "number",
+      name: options.name ?? "",
+      description: options.description,
+      position: options.position,
+      required: options.required ?? options.defaultValue === undefined,
+      defaultValue: options.defaultValue,
+    }),
+  };
+}
+
+function createOperationInputBuilder<Input extends object>(
+  fields: readonly WorkflowInputFieldDefinition[],
+): WorkflowOperationInputBuilder<Input> {
+  return {
+    fields,
+    extend(shape: WorkflowInputShape) {
+      const nextFields = Object.entries(shape).map(([name, field]) => ({
+        ...field,
+        name: field.name || name,
+      }));
+      return createOperationInputBuilder([...fields, ...nextFields]);
+    },
+  } as WorkflowOperationInputBuilder<Input>;
+}
+
+function createTask<Providers extends WorkflowProviderMap, InputContext extends JsonObject>(
+  app: WorkflowDefinition<string, Providers>,
+  name: string,
+  optionsOrHandler: WorkflowTaskOptions | WorkflowTaskHandler<Providers, InputContext, any>,
+  maybeHandler?: WorkflowTaskHandler<Providers, InputContext, any>,
+): WorkflowTaskNode<Providers, InputContext, any> {
+  const options = typeof optionsOrHandler === "function" ? undefined : optionsOrHandler;
+  const handler = (typeof optionsOrHandler === "function" ? optionsOrHandler : maybeHandler) as
+    | WorkflowTaskHandler<Providers, InputContext, any>
+    | undefined;
+  if (!handler) throw new Error(`Task ${name} is missing a handler`);
+
+  return {
+    kind: "rigkit.workflow-node",
+    nodeKind: "task",
+    name,
+    workflow: app,
+    options,
+    handler,
+  };
+}
+
+function validateProviders(providers: WorkflowProviderMap): void {
+  for (const [name, provider] of Object.entries(providers)) {
+    if (reservedTaskContextKeys.has(name)) {
+      throw new Error(`Provider name ${name} is reserved by the task context`);
+    }
+    if (!isProviderDefinition(provider)) {
+      throw new Error(`Provider ${name} is not a valid Rigkit provider`);
+    }
+  }
+}
+
+function assertSameWorkflow(
+  app: WorkflowDefinition<string, any>,
+  node: WorkflowNodeDefinition<any, any, any>,
+): void {
+  if (node.workflow !== app) {
+    throw new Error(`Node ${node.name} belongs to a different workflow`);
+  }
+}
+
+function getKind(value: object): unknown {
+  return (value as { kind?: unknown }).kind;
+}

package/src/db/index.ts

@@ -0,0 +1,157 @@
+import { mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import { Database } from "bun:sqlite";
+import { drizzle, type BunSQLiteDatabase } from "drizzle-orm/bun-sqlite";
+import { coreSchema, type CoreSchema } from "./schema/index.ts";
+
+export const RIGKIT_STATE_SCHEMA_VERSION = "drizzle-push";
+
+export type RigkitDatabase<TSchema extends Record<string, unknown> = CoreSchema> =
+  BunSQLiteDatabase<TSchema> & { $client: Database };
+
+export type RigkitDatabaseSchema = Record<string, unknown>;
+
+export type CreateRigkitDatabaseOptions<TSchema extends RigkitDatabaseSchema = CoreSchema> = {
+  schema?: TSchema;
+};
+
+export type SchemaSyncResult = {
+  applied: string[];
+  schemaVersion: string;
+  statements: string[];
+  warnings: string[];
+  hasDataLoss: boolean;
+};
+
+type DrizzleKitBunSQLiteDatabase = Pick<RigkitDatabase<RigkitDatabaseSchema>, "all" | "run">;
+
+type PushSQLiteSchemaResult = {
+  hasDataLoss: boolean;
+  warnings: string[];
+  statementsToExecute: string[];
+  apply(): Promise<void>;
+};
+
+type PushSQLiteSchemaForBun = (
+  imports: RigkitDatabaseSchema,
+  drizzleInstance: DrizzleKitBunSQLiteDatabase,
+) => Promise<PushSQLiteSchemaResult>;
+
+export function createRigkitDatabase<TSchema extends RigkitDatabaseSchema = CoreSchema>(
+  path: string,
+  options: CreateRigkitDatabaseOptions<TSchema> = {},
+): RigkitDatabase<TSchema> {
+  mkdirSync(dirname(path), { recursive: true });
+  const db = drizzle(new Database(path, { create: true }), {
+    schema: options.schema ?? (coreSchema as unknown as TSchema),
+  });
+  db.run("PRAGMA journal_mode = WAL");
+  db.run("PRAGMA foreign_keys = ON");
+  return db;
+}
+
+export async function syncRigkitDatabaseSchema<TSchema extends RigkitDatabaseSchema>(
+  db: RigkitDatabase<TSchema>,
+  schema: TSchema,
+): Promise<SchemaSyncResult> {
+  try {
+    const result = await pushRigkitDatabaseSchema(db, schema);
+    return toSchemaSyncResult(result);
+  } catch (error) {
+    const resetStatements = resetRigkitDatabase(db);
+    const result = await pushRigkitDatabaseSchema(db, schema);
+    return toSchemaSyncResult(result, {
+      resetStatements,
+      resetReason: errorMessage(error),
+    });
+  }
+}
+
+async function pushRigkitDatabaseSchema<TSchema extends RigkitDatabaseSchema>(
+  db: RigkitDatabase<TSchema>,
+  schema: TSchema,
+): Promise<PushSQLiteSchemaResult> {
+  const drizzleKitApi = ["drizzle-kit", "api"].join("/");
+  const { pushSQLiteSchema } = await import(drizzleKitApi);
+  const pushSchema = pushSQLiteSchema as unknown as PushSQLiteSchemaForBun;
+  const result = await silenceStdout(() => pushSchema(schema, db));
+  await silenceStdout(() => result.apply());
+  return result;
+}
+
+function toSchemaSyncResult(
+  result: PushSQLiteSchemaResult,
+  reset?: { resetStatements: string[]; resetReason: string },
+): SchemaSyncResult {
+  const statements = [...(reset?.resetStatements ?? []), ...result.statementsToExecute];
+  const warnings = [...result.warnings];
+  if (reset) {
+    warnings.unshift(`Reset Rigkit state database after Drizzle push failed: ${reset.resetReason}`);
+  }
+  return {
+    applied: statements.length > 0 ? [RIGKIT_STATE_SCHEMA_VERSION] : [],
+    schemaVersion: RIGKIT_STATE_SCHEMA_VERSION,
+    statements,
+    warnings,
+    hasDataLoss: reset !== undefined || result.hasDataLoss,
+  };
+}
+
+function resetRigkitDatabase<TSchema extends RigkitDatabaseSchema>(db: RigkitDatabase<TSchema>): string[] {
+  const rows = db.$client
+    .query(`
+      SELECT type, name
+      FROM sqlite_schema
+      WHERE type IN ('table', 'view', 'trigger')
+        AND name NOT LIKE 'sqlite_%'
+      ORDER BY CASE type
+        WHEN 'view' THEN 0
+        WHEN 'trigger' THEN 1
+        ELSE 2
+      END
+    `)
+    .all() as Array<{ type: "table" | "view" | "trigger"; name: string }>;
+
+  const statements = [
+    "PRAGMA foreign_keys=OFF",
+    ...rows.map((row) => `DROP ${row.type.toUpperCase()} IF EXISTS ${quoteSqlIdentifier(row.name)}`),
+    "PRAGMA foreign_keys=ON",
+  ];
+  const dropStatements = statements.slice(1, -1);
+  const reset = db.$client.transaction(() => {
+    for (const sql of dropStatements) {
+      db.$client.run(sql);
+    }
+  });
+  db.$client.run("PRAGMA foreign_keys=OFF");
+  try {
+    reset();
+  } finally {
+    db.$client.run("PRAGMA foreign_keys=ON");
+  }
+  return statements;
+}
+
+function quoteSqlIdentifier(name: string): string {
+  return `"${name.replaceAll('"', '""')}"`;
+}
+
+async function silenceStdout<T>(run: () => Promise<T>): Promise<T> {
+  const write = process.stdout.write;
+  process.stdout.write = (() => true) as typeof process.stdout.write;
+  try {
+    return await run();
+  } finally {
+    process.stdout.write = write;
+  }
+}
+
+function errorMessage(error: unknown): string {
+  if (error instanceof Error) return error.message;
+  if (typeof error === "string") return error;
+  try {
+    return JSON.stringify(error);
+  } catch {
+    return String(error);
+  }
+}

package/src/db/schema/core.ts

@@ -0,0 +1,75 @@
+import { index, integer, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core";
+import type { JsonValue } from "../../types.ts";
+
+export const workspaces = sqliteTable(
+  "workspaces",
+  {
+    id: text("id").primaryKey(),
+    name: text("name").notNull(),
+    providerId: text("provider_id").notNull(),
+    workflow: text("workflow").notNull(),
+    resourceId: text("resource_id").notNull(),
+    snapshotId: text("snapshot_id"),
+    sourceRef: text("source_ref_json", { mode: "json" }).$type<JsonValue>().notNull(),
+    context: text("context_json", { mode: "json" }).$type<Record<string, JsonValue>>().notNull(),
+    createdAt: text("created_at").notNull(),
+    updatedAt: text("updated_at").notNull(),
+    metadata: text("metadata_json", { mode: "json" }).$type<Record<string, JsonValue>>().notNull(),
+  },
+  (table) => [
+    uniqueIndex("workspaces_name_idx").on(table.name),
+    index("workspaces_provider_resource_idx").on(table.providerId, table.resourceId),
+  ],
+);
+
+export const workflowNodeRuns = sqliteTable(
+  "workflow_node_runs",
+  {
+    id: text("id").primaryKey(),
+    workflow: text("workflow").notNull(),
+    nodePath: text("node_path").notNull(),
+    nodeName: text("node_name").notNull(),
+    nodeKind: text("node_kind").notNull(),
+    nodeKey: text("node_key").notNull(),
+    providerFingerprint: text("provider_fingerprint").notNull(),
+    upstreamRunIds: text("upstream_run_ids_json", { mode: "json" }).$type<string[]>().notNull(),
+    output: text("output_json", { mode: "json" }).$type<Record<string, JsonValue>>().notNull(),
+    artifacts: text("artifacts_json", { mode: "json" }).$type<JsonValue[]>().notNull(),
+    invalidated: integer("invalidated", { mode: "boolean" }).notNull().default(false),
+    createdAt: text("created_at").notNull(),
+    metadata: text("metadata_json", { mode: "json" }).$type<Record<string, JsonValue>>().notNull(),
+  },
+  (table) => [
+    index("workflow_node_runs_lookup_idx").on(
+      table.workflow,
+      table.nodePath,
+      table.nodeKey,
+      table.providerFingerprint,
+    ),
+    index("workflow_node_runs_created_idx").on(table.createdAt),
+  ],
+);
+
+export const providerState = sqliteTable(
+  "provider_state",
+  {
+    providerId: text("provider_id").notNull(),
+    key: text("key").notNull(),
+    value: text("value_json", { mode: "json" }).$type<JsonValue>().notNull(),
+    createdAt: text("created_at").notNull(),
+    updatedAt: text("updated_at").notNull(),
+  },
+  (table) => [
+    uniqueIndex("provider_state_provider_key_idx").on(table.providerId, table.key),
+    index("provider_state_provider_idx").on(table.providerId),
+  ],
+);
+
+export const runtimeMetadata = sqliteTable(
+  "runtime_metadata",
+  {
+    key: text("key").primaryKey(),
+    value: text("value_json", { mode: "json" }).$type<JsonValue>().notNull(),
+    updatedAt: text("updated_at").notNull(),
+  },
+);

package/src/db/schema/index.ts

@@ -0,0 +1,7 @@
+import * as core from "./core.ts";
+
+export const coreSchema = core;
+
+export type CoreSchema = typeof coreSchema;
+
+export * from "./core.ts";