@memgrafter/flatagents 0.8.3 → 0.9.0

package/MACHINES.md

@@ -22,7 +22,7 @@
 ```yaml
 # profiles.yml — agents reference by name
 spec: flatprofiles
-spec_version: "0.8.3"
+spec_version: "0.9.0"
 data:
   model_profiles:
     fast: { provider: cerebras, name: zai-glm-4.6, temperature: 0.6 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@memgrafter/flatagents",
-  "version": "0.8.3",
+  "version": "0.9.0",
   "description": "TypeScript SDK for FlatAgents - Declarative LLM orchestration with YAML",
   "main": "dist/index.js",
   "module": "dist/index.mjs",

package/schemas/flatagent.d.ts

@@ -149,7 +149,7 @@
  * The profile field specifies which profile name to use as base.
  */
 
-export const SPEC_VERSION = "0.8.3";
+export const SPEC_VERSION = "0.9.0";
 
 export interface AgentWrapper {
   spec: "flatagent";

package/schemas/flatagent.slim.d.ts

@@ -1,4 +1,4 @@
-export const SPEC_VERSION = "0.8.3";
+export const SPEC_VERSION = "0.9.0";
 export interface AgentWrapper {
     spec: "flatagent";
     spec_version: string;

package/schemas/flatagents-runtime.d.ts

@@ -13,6 +13,8 @@
  *   - ResultBackend: InMemoryResultBackend (MUST)
  *   - ExecutionType: Default, Retry, Parallel, MDAPVoting (MUST)
  *   - MachineHooks: Base interface (MUST)
+ *   - RegistrationBackend: SQLiteRegistrationBackend (MUST), MemoryRegistrationBackend (SHOULD)
+ *   - WorkBackend: SQLiteWorkBackend (MUST), MemoryWorkBackend (SHOULD)
  *
  * OPTIONAL IMPLEMENTATIONS:
  * -------------------------
@@ -314,6 +316,158 @@ export interface LaunchIntent {
     launched: boolean;
 }
 
+/**
+ * REGISTRATION BACKEND:
+ * ---------------------
+ * Worker lifecycle management for distributed execution.
+ *
+ * SDKs MUST provide:
+ *   - SQLiteRegistrationBackend: For local deployments
+ *
+ * SDKs SHOULD provide:
+ *   - MemoryRegistrationBackend: For testing
+ *
+ * Implementation notes:
+ *   - Time units: Python reference SDK uses seconds for all interval values
+ *   - Stale threshold: SDKs SHOULD default to 2× heartbeat_interval if not specified
+ */
+export interface RegistrationBackend {
+    /**
+     * Register a new worker.
+     * Creates a new worker record with status "active".
+     */
+    register(worker: WorkerRegistration): Promise<WorkerRecord>;
+
+    /**
+     * Update worker's last_heartbeat timestamp.
+     * Can optionally update metadata.
+     */
+    heartbeat(worker_id: string, metadata?: Record<string, any>): Promise<void>;
+
+    /**
+     * Update worker status.
+     * Status values (string, not enum for extensibility):
+     *   - "active": Worker is running and healthy
+     *   - "terminating": Worker received shutdown signal
+     *   - "terminated": Worker exited cleanly
+     *   - "lost": Worker failed heartbeat, presumed dead
+     */
+    updateStatus(worker_id: string, status: string): Promise<void>;
+
+    /**
+     * Get a worker record by ID.
+     * @returns The worker record, or null if not found
+     */
+    get(worker_id: string): Promise<WorkerRecord | null>;
+
+    /**
+     * List workers matching filter criteria.
+     */
+    list(filter?: WorkerFilter): Promise<WorkerRecord[]>;
+}
+
+export interface WorkerRegistration {
+    worker_id: string;
+    host?: string;
+    pid?: number;
+    capabilities?: string[];  // e.g., ["gpu", "paper-analysis"]
+    pool_id?: string;         // Worker pool grouping
+    started_at: string;
+}
+
+export interface WorkerRecord extends WorkerRegistration {
+    status: string;           // See status values in RegistrationBackend.updateStatus
+    last_heartbeat: string;
+    current_task_id?: string;
+}
+
+export interface WorkerFilter {
+    status?: string | string[];
+    capability?: string;
+    pool_id?: string;
+    stale_threshold_seconds?: number;  // Filter workers with old heartbeats
+}
+
+/**
+ * WORK BACKEND:
+ * -------------
+ * Work distribution via named pools with atomic claim.
+ *
+ * SDKs MUST provide:
+ *   - SQLiteWorkBackend: For local deployments
+ *
+ * SDKs SHOULD provide:
+ *   - MemoryWorkBackend: For testing
+ *
+ * Implementation notes:
+ *   - Atomic claim: SDKs MUST ensure no two workers can claim the same job
+ *   - Test requirements: Include concurrent claim race condition tests
+ */
+export interface WorkBackend {
+    /**
+     * Get a named work pool.
+     * Creates the pool if it doesn't exist.
+     */
+    pool(name: string): WorkPool;
+}
+
+export interface WorkPool {
+    /**
+     * Add work item to the pool.
+     * @param item - The work data (will be JSON serialized)
+     * @param options.max_retries - Max retry attempts before poisoning (default: 3)
+     * @returns The item ID
+     */
+    push(item: any, options?: { max_retries?: number }): Promise<string>;
+
+    /**
+     * Atomically claim next available item.
+     * MUST be atomic - no two workers can claim the same job.
+     * @returns The claimed item, or null if pool is empty
+     */
+    claim(worker_id: string): Promise<WorkItem | null>;
+
+    /**
+     * Mark item as complete.
+     * Sets status to "done" and stores result.
+     */
+    complete(item_id: string, result?: any): Promise<void>;
+
+    /**
+     * Mark item as failed.
+     * Increments attempts. If attempts >= max_retries, marks as "poisoned".
+     * Otherwise returns to "pending" status for retry.
+     */
+    fail(item_id: string, error?: string): Promise<void>;
+
+    /**
+     * Get pool depth (unclaimed pending items).
+     */
+    size(): Promise<number>;
+
+    /**
+     * Release all jobs claimed by a worker.
+     * Used for stale worker cleanup.
+     * @returns Number of jobs released
+     */
+    releaseByWorker(worker_id: string): Promise<number>;
+}
+
+export interface WorkItem {
+    id: string;
+    data: any;
+    claimed_by?: string;
+    claimed_at?: string;
+    attempts: number;
+    max_retries: number;  // default: 3
+}
+
+// Job status values (string):
+// - "pending": Available for claim
+// - "claimed": Currently being processed
+// - "done": Successfully completed
+// - "poisoned": Failed max_retries times, will not be retried
+
 export interface BackendConfig {
     /** Checkpoint storage. Default: memory */
     persistence?: "memory" | "local" | "redis" | "postgres" | "s3";
@@ -323,24 +477,36 @@ export interface BackendConfig {
 
     /** Inter-machine results. Default: memory */
     results?: "memory" | "redis";
+
+    /** Worker registration. Default: memory */
+    registration?: "memory" | "sqlite" | "redis";
+
+    /** Work pool. Default: memory */
+    work?: "memory" | "sqlite" | "redis";
+
+    /** Path for sqlite backends (registration and work share this) */
+    sqlite_path?: string;
 }
 
-export const SPEC_VERSION = "0.8.3";
+export const SPEC_VERSION = "0.9.0";
 
 /**
  * Wrapper interface for JSON schema generation.
  * Groups all runtime interfaces that SDKs must implement.
  */
 export interface SDKRuntimeWrapper {
-  spec: "flatagents-runtime";
-  spec_version: typeof SPEC_VERSION;
-  execution_lock?: ExecutionLock;
-  persistence_backend?: PersistenceBackend;
-  result_backend?: ResultBackend;
-  execution_config?: ExecutionConfig;
-  machine_hooks?: MachineHooks;
-  llm_backend?: LLMBackend;
-  machine_invoker?: MachineInvoker;
-  backend_config?: BackendConfig;
-  machine_snapshot?: MachineSnapshot;
+    spec: "flatagents-runtime";
+    spec_version: typeof SPEC_VERSION;
+    execution_lock?: ExecutionLock;
+    persistence_backend?: PersistenceBackend;
+    result_backend?: ResultBackend;
+    execution_config?: ExecutionConfig;
+    machine_hooks?: MachineHooks;
+    llm_backend?: LLMBackend;
+    machine_invoker?: MachineInvoker;
+    backend_config?: BackendConfig;
+    machine_snapshot?: MachineSnapshot;
+    registration_backend?: RegistrationBackend;
+    work_backend?: WorkBackend;
 }
+

package/schemas/flatagents-runtime.schema.json

@@ -11,7 +11,7 @@
         },
         "spec_version": {
           "type": "string",
-          "const": "0.8.3"
+          "const": "0.9.0"
         },
         "execution_lock": {
           "$ref": "#/definitions/ExecutionLock"
@@ -39,6 +39,12 @@
         },
         "machine_snapshot": {
           "$ref": "#/definitions/MachineSnapshot"
+        },
+        "registration_backend": {
+          "$ref": "#/definitions/RegistrationBackend"
+        },
+        "work_backend": {
+          "$ref": "#/definitions/WorkBackend"
         }
       },
       "required": [
@@ -51,7 +57,7 @@
     "ExecutionLock": {
       "type": "object",
       "additionalProperties": false,
-      "description": "FlatAgents Runtime Interface Spec ==================================\n\nThis file defines the runtime interfaces that SDKs MUST implement to be considered compliant. These are NOT configuration schemas (see flatagent.d.ts and flatmachine.d.ts for those).\n\nREQUIRED IMPLEMENTATIONS:\n-------------------------   - ExecutionLock: NoOpLock (MUST), LocalFileLock (SHOULD)   - PersistenceBackend: MemoryBackend (MUST), LocalFileBackend (SHOULD)   - ResultBackend: InMemoryResultBackend (MUST)   - ExecutionType: Default, Retry, Parallel, MDAPVoting (MUST)   - MachineHooks: Base interface (MUST)\n\nOPTIONAL IMPLEMENTATIONS:\n-------------------------   - Distributed backends (Redis, Postgres, etc.)   - LLMBackend (SDK may use native provider SDKs)\n\nEXECUTION LOCKING:\n------------------ Prevents concurrent execution of the same machine instance.\n\nSDKs MUST provide:   - NoOpLock: For when locking is handled externally or disabled\n\nSDKs SHOULD provide:   - LocalFileLock: For single-node deployments using fcntl/flock\n\nDistributed deployments should implement Redis/Consul/etcd locks.\n\nPERSISTENCE BACKEND:\n-------------------- Storage backend for machine checkpoints.\n\nSDKs MUST provide:   - MemoryBackend: For testing and ephemeral runs\n\nSDKs SHOULD provide:   - LocalFileBackend: For durable local storage with atomic writes\n\nRESULT BACKEND:\n--------------- Inter-machine communication via URI-addressed results.\n\nURI format: flatagents://{execution_id}/{path}   - path is typically \"result\" or \"checkpoint\"\n\nSDKs MUST provide:   - InMemoryResultBackend: For single-process execution\n\nEXECUTION TYPES:\n---------------- Execution strategy for agent calls.\n\nSDKs MUST implement all four types:   - default: Single call, no retry   - retry: Configurable backoffs with jitter   - parallel: Run N samples, return all successes   - mdap_voting: Multi-sample with consensus voting\n\nMACHINE HOOKS:\n-------------- Extension points for machine execution. All methods are optional and can be sync or async.\n\nSDKs SHOULD provide:   - WebhookHooks: Send events to HTTP endpoint   - CompositeHooks: Combine multiple hook implementations\n\nLLM BACKEND (OPTIONAL):\n----------------------- Abstraction over LLM providers.\n\nThis interface is OPTIONAL - SDKs may use provider SDKs directly. Useful for:   - Unified retry/monitoring across providers   - Provider-agnostic code   - Testing with mock backends\n\nMACHINE INVOKER:\n---------------- Interface for invoking peer machines. Used internally by FlatMachine for `machine:` and `launch:` states.\n\nBACKEND CONFIGURATION:\n---------------------- Backend configuration for machine settings.\n\nExample in YAML:   settings:     backends:       persistence: local       locking: none       results: memory"
+      "description": "FlatAgents Runtime Interface Spec ==================================\n\nThis file defines the runtime interfaces that SDKs MUST implement to be considered compliant. These are NOT configuration schemas (see flatagent.d.ts and flatmachine.d.ts for those).\n\nREQUIRED IMPLEMENTATIONS:\n-------------------------   - ExecutionLock: NoOpLock (MUST), LocalFileLock (SHOULD)   - PersistenceBackend: MemoryBackend (MUST), LocalFileBackend (SHOULD)   - ResultBackend: InMemoryResultBackend (MUST)   - ExecutionType: Default, Retry, Parallel, MDAPVoting (MUST)   - MachineHooks: Base interface (MUST)   - RegistrationBackend: SQLiteRegistrationBackend (MUST), MemoryRegistrationBackend (SHOULD)   - WorkBackend: SQLiteWorkBackend (MUST), MemoryWorkBackend (SHOULD)\n\nOPTIONAL IMPLEMENTATIONS:\n-------------------------   - Distributed backends (Redis, Postgres, etc.)   - LLMBackend (SDK may use native provider SDKs)\n\nEXECUTION LOCKING:\n------------------ Prevents concurrent execution of the same machine instance.\n\nSDKs MUST provide:   - NoOpLock: For when locking is handled externally or disabled\n\nSDKs SHOULD provide:   - LocalFileLock: For single-node deployments using fcntl/flock\n\nDistributed deployments should implement Redis/Consul/etcd locks.\n\nPERSISTENCE BACKEND:\n-------------------- Storage backend for machine checkpoints.\n\nSDKs MUST provide:   - MemoryBackend: For testing and ephemeral runs\n\nSDKs SHOULD provide:   - LocalFileBackend: For durable local storage with atomic writes\n\nRESULT BACKEND:\n--------------- Inter-machine communication via URI-addressed results.\n\nURI format: flatagents://{execution_id}/{path}   - path is typically \"result\" or \"checkpoint\"\n\nSDKs MUST provide:   - InMemoryResultBackend: For single-process execution\n\nEXECUTION TYPES:\n---------------- Execution strategy for agent calls.\n\nSDKs MUST implement all four types:   - default: Single call, no retry   - retry: Configurable backoffs with jitter   - parallel: Run N samples, return all successes   - mdap_voting: Multi-sample with consensus voting\n\nMACHINE HOOKS:\n-------------- Extension points for machine execution. All methods are optional and can be sync or async.\n\nSDKs SHOULD provide:   - WebhookHooks: Send events to HTTP endpoint   - CompositeHooks: Combine multiple hook implementations\n\nLLM BACKEND (OPTIONAL):\n----------------------- Abstraction over LLM providers.\n\nThis interface is OPTIONAL - SDKs may use provider SDKs directly. Useful for:   - Unified retry/monitoring across providers   - Provider-agnostic code   - Testing with mock backends\n\nMACHINE INVOKER:\n---------------- Interface for invoking peer machines. Used internally by FlatMachine for `machine:` and `launch:` states.\n\nBACKEND CONFIGURATION:\n---------------------- Backend configuration for machine settings.\n\nExample in YAML:   settings:     backends:       persistence: local       locking: none       results: memory"
     },
     "PersistenceBackend": {
       "type": "object",
@@ -154,6 +160,28 @@
             "redis"
           ],
           "description": "Inter-machine results. Default: memory"
+        },
+        "registration": {
+          "type": "string",
+          "enum": [
+            "memory",
+            "sqlite",
+            "redis"
+          ],
+          "description": "Worker registration. Default: memory"
+        },
+        "work": {
+          "type": "string",
+          "enum": [
+            "memory",
+            "sqlite",
+            "redis"
+          ],
+          "description": "Work pool. Default: memory"
+        },
+        "sqlite_path": {
+          "type": "string",
+          "description": "Path for sqlite backends (registration and work share this)"
         }
       },
       "additionalProperties": false
@@ -238,6 +266,16 @@
         "launched"
       ],
       "additionalProperties": false
+    },
+    "RegistrationBackend": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "REGISTRATION BACKEND:\n--------------------- Worker lifecycle management for distributed execution.\n\nSDKs MUST provide:   - SQLiteRegistrationBackend: For local deployments\n\nSDKs SHOULD provide:   - MemoryRegistrationBackend: For testing\n\nImplementation notes:   - Time units: Python reference SDK uses seconds for all interval values   - Stale threshold: SDKs SHOULD default to 2× heartbeat_interval if not specified"
+    },
+    "WorkBackend": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "WORK BACKEND:\n------------- Work distribution via named pools with atomic claim.\n\nSDKs MUST provide:   - SQLiteWorkBackend: For local deployments\n\nSDKs SHOULD provide:   - MemoryWorkBackend: For testing\n\nImplementation notes:   - Atomic claim: SDKs MUST ensure no two workers can claim the same job   - Test requirements: Include concurrent claim race condition tests"
     }
   }
 }

package/schemas/flatagents-runtime.slim.d.ts

@@ -102,12 +102,62 @@ export interface LaunchIntent {
     input: Record<string, any>;
     launched: boolean;
 }
+export interface RegistrationBackend {
+    register(worker: WorkerRegistration): Promise<WorkerRecord>;
+    heartbeat(worker_id: string, metadata?: Record<string, any>): Promise<void>;
+    updateStatus(worker_id: string, status: string): Promise<void>;
+    get(worker_id: string): Promise<WorkerRecord | null>;
+    list(filter?: WorkerFilter): Promise<WorkerRecord[]>;
+}
+export interface WorkerRegistration {
+    worker_id: string;
+    host?: string;
+    pid?: number;
+    capabilities?: string[];
+    pool_id?: string;
+    started_at: string;
+}
+export interface WorkerRecord extends WorkerRegistration {
+    status: string;
+    last_heartbeat: string;
+    current_task_id?: string;
+}
+export interface WorkerFilter {
+    status?: string | string[];
+    capability?: string;
+    pool_id?: string;
+    stale_threshold_seconds?: number;
+}
+export interface WorkBackend {
+    pool(name: string): WorkPool;
+}
+export interface WorkPool {
+    push(item: any, options?: {
+        max_retries?: number;
+    }): Promise<string>;
+    claim(worker_id: string): Promise<WorkItem | null>;
+    complete(item_id: string, result?: any): Promise<void>;
+    fail(item_id: string, error?: string): Promise<void>;
+    size(): Promise<number>;
+    releaseByWorker(worker_id: string): Promise<number>;
+}
+export interface WorkItem {
+    id: string;
+    data: any;
+    claimed_by?: string;
+    claimed_at?: string;
+    attempts: number;
+    max_retries: number;
+}
 export interface BackendConfig {
     persistence?: "memory" | "local" | "redis" | "postgres" | "s3";
     locking?: "none" | "local" | "redis" | "consul";
     results?: "memory" | "redis";
+    registration?: "memory" | "sqlite" | "redis";
+    work?: "memory" | "sqlite" | "redis";
+    sqlite_path?: string;
 }
-export const SPEC_VERSION = "0.8.3";
+export const SPEC_VERSION = "0.9.0";
 export interface SDKRuntimeWrapper {
     spec: "flatagents-runtime";
     spec_version: typeof SPEC_VERSION;
@@ -120,4 +170,6 @@ export interface SDKRuntimeWrapper {
     machine_invoker?: MachineInvoker;
     backend_config?: BackendConfig;
     machine_snapshot?: MachineSnapshot;
+    registration_backend?: RegistrationBackend;
+    work_backend?: WorkBackend;
 }

package/schemas/flatmachine.d.ts

@@ -256,7 +256,7 @@
  * pending_launches    - Outbox pattern (v0.4.0)
  */
 
-export const SPEC_VERSION = "0.8.3";
+export const SPEC_VERSION = "0.9.0";
 
 export interface MachineWrapper {
   spec: "flatmachine";

package/schemas/flatmachine.slim.d.ts

@@ -1,4 +1,4 @@
-export const SPEC_VERSION = "0.8.3";
+export const SPEC_VERSION = "0.9.0";
 export interface MachineWrapper {
     spec: "flatmachine";
     spec_version: string;

package/schemas/profiles.d.ts

@@ -107,7 +107,7 @@
  * base_url          - Custom base URL for the API (e.g., for local models or proxies)
  */
 
-export const SPEC_VERSION = "0.8.3";
+export const SPEC_VERSION = "0.9.0";
 
 export interface ProfilesWrapper {
   spec: "flatprofiles";

package/schemas/profiles.slim.d.ts

@@ -1,4 +1,4 @@
-export const SPEC_VERSION = "0.8.3";
+export const SPEC_VERSION = "0.9.0";
 export interface ProfilesWrapper {
     spec: "flatprofiles";
     spec_version: string;