colonies-ts 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Johan Kristiansson, RISE Research Institutes of Sweden AB 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,341 @@
+# Colonies TS Client Library
+
+[![CI](https://github.com/colonyos/colonies-ts/actions/workflows/node.yml/badge.svg)](https://github.com/colonyos/colonies-ts/actions/workflows/node.yml)
+[![codecov](https://img.shields.io/codecov/c/github/colonyos/colonies-ts)](https://codecov.io/gh/colonyos/colonies-ts)
+[![npm version](https://badge.fury.io/js/colonies-ts.svg)](https://www.npmjs.com/package/colonies-ts)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+TypeScript client library for ColonyOS - a distributed meta-orchestrator for compute continuums.
+
+## Installation
+
+```bash
+npm install colonies-ts
+```
+
+## Three Execution Patterns
+
+ColonyOS supports three patterns for distributed task execution:
+
+### 1. Batch Processing
+
+Traditional request-response pattern for discrete tasks. Submit a job, an executor picks it up, processes it, and returns the result.
+
+```mermaid
+sequenceDiagram
+    participant Client
+    participant Colonies Server
+    participant Executor
+
+    Client->>Colonies Server: submitFunctionSpec()
+    Colonies Server-->>Client: Process (WAITING)
+    Client->>Colonies Server: subscribeProcess(SUCCESS)
+
+    Executor->>Colonies Server: assign()
+    Colonies Server-->>Executor: Process (RUNNING)
+
+    Note over Executor: Execute task
+
+    Executor->>Colonies Server: closeProcess(output)
+    Colonies Server-->>Client: Process (SUCCESS)
+```
+
+```typescript
+// Submit a batch job
+const process = await client.submitFunctionSpec({
+  funcname: 'process-image',
+  kwargs: { imageUrl: 'https://example.com/image.jpg' },
+  conditions: {
+    colonyname: 'my-colony',
+    executortype: 'image-processor',
+  },
+  maxexectime: 300,
+});
+
+// Subscribe to process completion
+client.subscribeProcess(
+  'my-colony', process.processid, ProcessState.SUCCESS, 300,
+  (result) => console.log('Output:', result.output),
+  console.error,
+  () => {}
+);
+```
+
+### 2. Blueprint Reconciliation
+
+Declarative desired-state pattern for managing resources. Define the desired state in a blueprint, and a reconciler continuously ensures the actual state matches.
+
+```mermaid
+sequenceDiagram
+    participant User
+    participant Colonies Server
+    participant Reconciler
+    participant Device
+
+    User->>Colonies Server: updateBlueprint(spec)
+    Note over Colonies Server: Generation++
+
+    Colonies Server->>Colonies Server: Create reconcile process
+    Reconciler->>Colonies Server: assign()
+    Colonies Server-->>Reconciler: Process (RUNNING)
+
+    Reconciler->>Colonies Server: getBlueprint()
+    Colonies Server-->>Reconciler: spec + status
+
+    Reconciler->>Device: Apply changes
+    Device-->>Reconciler: Current state
+
+    Reconciler->>Colonies Server: updateBlueprintStatus()
+    Reconciler->>Colonies Server: closeProcess()
+    Note over Colonies Server: spec == status
+```
+
+```typescript
+// Create a blueprint with desired state
+await client.addBlueprint({
+  kind: 'HomeDevice',
+  metadata: { name: 'living-room-light', colonyname: 'home' },
+  handler: { executortype: 'home-reconciler' },
+  spec: { power: true, brightness: 80 },  // Desired state
+});
+
+// Update desired state - reconciler will sync
+const bp = await client.getBlueprint('home', 'living-room-light');
+bp.spec.brightness = 50;
+await client.updateBlueprint(bp);
+
+// Or via CLI: colonies blueprint set --name living-room-light --key spec.brightness --value 50
+```
+
+### 3. Real-time Channels
+
+Bidirectional streaming for interactive workloads like chat, live data, or long-running processes with progress updates.
+
+```mermaid
+sequenceDiagram
+    participant Client
+    participant Colonies Server
+    participant Executor
+
+    Client->>Colonies Server: submitFunctionSpec(channels)
+    Colonies Server-->>Client: Process (WAITING)
+    Client->>Colonies Server: subscribeProcess(RUNNING)
+
+    Executor->>Colonies Server: assign()
+    Colonies Server-->>Executor: Process (RUNNING)
+    Colonies Server-->>Client: Process (RUNNING)
+
+    Client->>Colonies Server: subscribeChannel()
+    Note over Client,Executor: WebSocket streams open
+
+    Client->>Colonies Server: channelAppend("prompt")
+    Colonies Server-->>Executor: Message
+
+    loop Streaming response
+        Executor->>Colonies Server: channelAppend("token")
+        Colonies Server-->>Client: Message
+    end
+
+    Executor->>Colonies Server: closeProcess()
+```
+
+```typescript
+// Submit process with channel
+const process = await client.submitFunctionSpec({
+  funcname: 'chat',
+  kwargs: { model: 'llama3' },
+  conditions: { colonyname: 'ai', executortype: 'llm' },
+  channels: ['chat'],
+});
+
+// Wait for process to be assigned, then subscribe to channel
+client.subscribeProcess(
+  'ai', process.processid, ProcessState.RUNNING, 60,
+  (runningProcess) => {
+    // Now subscribe to channel for streaming
+    client.subscribeChannel(
+      runningProcess.processid, 'chat', 0, 300,
+      (entries) => entries.forEach(e => console.log(e.payload)),
+      console.error,
+      () => {}
+    );
+    // Send message
+    client.channelAppend(runningProcess.processid, 'chat', 1, 0, 'Hello!');
+  },
+  console.error,
+  () => {}
+);
+```
+
+## Crypto
+
+The library includes a self-contained secp256k1 ECDSA implementation:
+
+```typescript
+import { Crypto } from 'colonies-ts';
+
+const crypto = new Crypto();
+
+// Generate a new private key
+const privateKey = crypto.generatePrivateKey();
+
+// Derive the public ID from a private key
+const id = crypto.id(privateKey);
+
+// Sign a message
+const signature = crypto.sign('message', privateKey);
+```
+
+## Examples
+
+- [Home Automation](examples/blueprint/) - Complete web app for managing smart home devices using blueprints
+
+## Documentation
+
+- [Getting Started](docs/getting-started.md) - Introduction to ColonyOS and basic usage
+- [Using Channels](docs/channels.md) - Real-time messaging between clients and executors
+- [Building Reconcilers](docs/reconciler.md) - Blueprint and reconciler tutorial
+- [API Reference](docs/api-reference.md) - Complete API documentation
+
+## Development
+
+### Prerequisites
+
+- Node.js >= 18
+- npm
+
+### Install dependencies
+
+```bash
+npm install
+```
+
+### Run tests
+
+```bash
+npm test                 # Unit tests
+npm run test:integration # Integration tests (requires running server)
+npm run test:all         # All tests
+```
+
+Integration tests require a running ColonyOS server:
+
+```bash
+cd /path/to/colonies
+docker-compose up -d
+```
+
+### Build
+
+```bash
+npm run build
+```
+
+This generates ESM and CommonJS builds in the `dist/` directory.
+
+## API Reference
+
+### ColoniesClient
+
+```typescript
+new ColoniesClient({
+  host: string,      // Server hostname
+  port: number,      // Server port
+  tls?: boolean,     // Enable TLS (default: false)
+})
+```
+
+#### Colony & Server
+
+| Method | Description |
+|--------|-------------|
+| `setPrivateKey(key)` | Set the private key for signing requests |
+| `getColonies()` | List all colonies |
+| `getStatistics()` | Get server statistics |
+| `addColony(colony)` | Add a new colony |
+| `removeColony(colonyName)` | Remove a colony |
+
+#### Executors
+
+| Method | Description |
+|--------|-------------|
+| `getExecutors(colonyName)` | List executors in a colony |
+| `getExecutor(colonyName, executorName)` | Get a specific executor |
+| `addExecutor(executor)` | Register a new executor |
+| `approveExecutor(colonyName, executorName)` | Approve an executor |
+| `removeExecutor(colonyName, executorName)` | Remove an executor |
+
+#### Processes
+
+| Method | Description |
+|--------|-------------|
+| `submitFunctionSpec(spec)` | Submit a process |
+| `assign(colonyName, timeout, prvKey)` | Assign a process to execute |
+| `getProcess(processId)` | Get process details |
+| `getProcesses(colonyName, count, state)` | List processes by state |
+| `closeProcess(processId, output)` | Close a process successfully |
+| `failProcess(processId, errors)` | Close a process with failure |
+| `removeProcess(processId)` | Remove a process |
+| `removeAllProcesses(colonyName, state)` | Remove all processes |
+
+#### Workflows
+
+| Method | Description |
+|--------|-------------|
+| `submitWorkflowSpec(spec)` | Submit a workflow (DAG) |
+| `getProcessGraph(graphId)` | Get workflow details |
+| `getProcessGraphs(colonyName, count, state?)` | List workflows |
+| `removeProcessGraph(graphId)` | Remove a workflow |
+| `removeAllProcessGraphs(colonyName, state?)` | Remove all workflows |
+
+#### Channels
+
+| Method | Description |
+|--------|-------------|
+| `channelAppend(processId, channelName, seq, inReplyTo, payload)` | Send message to channel |
+| `channelRead(processId, channelName, afterSeq, limit)` | Read messages from channel |
+| `subscribeChannel(...)` | Subscribe to channel via WebSocket |
+| `subscribeProcess(...)` | Subscribe to process state changes |
+
+#### Blueprints
+
+| Method | Description |
+|--------|-------------|
+| `addBlueprintDefinition(definition)` | Add a blueprint definition |
+| `getBlueprintDefinition(colonyName, name)` | Get a blueprint definition |
+| `getBlueprintDefinitions(colonyName)` | List blueprint definitions |
+| `removeBlueprintDefinition(colonyName, name)` | Remove a blueprint definition |
+| `addBlueprint(blueprint)` | Add a blueprint |
+| `getBlueprint(colonyName, name)` | Get a blueprint |
+| `getBlueprints(colonyName, kind?, location?)` | List blueprints |
+| `updateBlueprint(blueprint, forceGeneration?)` | Update a blueprint |
+| `removeBlueprint(colonyName, name)` | Remove a blueprint |
+| `updateBlueprintStatus(colonyName, name, status)` | Update blueprint status |
+| `reconcileBlueprint(colonyName, name, force?)` | Trigger reconciliation |
+
+#### Crons & Generators
+
+| Method | Description |
+|--------|-------------|
+| `getCrons(colonyName)` | List cron jobs |
+| `getCron(cronId)` | Get a cron job |
+| `addCron(cronSpec)` | Add a cron job |
+| `removeCron(cronId)` | Remove a cron job |
+| `getGenerators(colonyName)` | List generators |
+| `getGenerator(generatorId)` | Get a generator |
+| `addGenerator(generatorSpec)` | Add a generator |
+
+### ProcessState
+
+```typescript
+enum ProcessState {
+  WAITING = 0,
+  RUNNING = 1,
+  SUCCESS = 2,
+  FAILED = 3,
+}
+```
+
+## License
+
+MIT

package/dist/index.d.mts

@@ -0,0 +1,274 @@
+/**
+ * Generate a new random private key
+ * @returns Hex-encoded private key (64 characters)
+ */
+declare function generatePrivateKey(): string;
+/**
+ * Derive the public ID from a private key
+ * Uses SHA3-256 hash of "04" + hex(publicKey)
+ * @param privateKey - Hex-encoded private key
+ * @returns Hex-encoded ID (64 characters)
+ */
+declare function deriveId(privateKey: string): string;
+/**
+ * Sign a message with a private key
+ * @param message - Message to sign
+ * @param privateKey - Hex-encoded private key
+ * @returns Hex-encoded signature (130 characters: r + s + v)
+ */
+declare function sign(message: string, privateKey: string): string;
+/**
+ * Crypto utility class for convenience
+ */
+declare class Crypto {
+    generatePrivateKey(): string;
+    id(privateKey: string): string;
+    sign(message: string, privateKey: string): string;
+}
+
+/**
+ * ColonyOS Client
+ * Ported from colonyspace/aila/src/lib/api/colony.ts
+ */
+interface ColoniesClientConfig {
+    host: string;
+    port: number;
+    tls?: boolean;
+}
+interface RPCMessage {
+    payloadtype: string;
+    payload: string;
+    signature: string;
+}
+interface FunctionSpec {
+    nodename?: string;
+    funcname: string;
+    args?: any[];
+    kwargs?: Record<string, any>;
+    priority?: number;
+    maxwaittime?: number;
+    maxexectime?: number;
+    maxretries?: number;
+    conditions?: {
+        colonyname?: string;
+        executornames?: string[];
+        executortype?: string;
+        dependencies?: string[];
+        nodes?: number;
+        cpu?: string;
+        processes?: number;
+        processespernode?: number;
+        mem?: string;
+        storage?: string;
+        gpu?: {
+            name?: string;
+            mem?: string;
+            count?: number;
+            nodecount?: number;
+        };
+        walltime?: number;
+    };
+    label?: string;
+    fs?: any;
+    env?: Record<string, string>;
+    channels?: string[];
+}
+declare enum ProcessState {
+    WAITING = 0,
+    RUNNING = 1,
+    SUCCESS = 2,
+    FAILED = 3
+}
+declare class ColoniesClient {
+    private host;
+    private port;
+    private tls;
+    private crypto;
+    private privateKey;
+    constructor(config: ColoniesClientConfig);
+    setPrivateKey(privateKey: string): void;
+    private getBaseUrl;
+    private createRPCMsg;
+    private sendRPC;
+    getColonies(): Promise<any>;
+    getStatistics(): Promise<any>;
+    /**
+     * Add a new colony (requires server private key)
+     * @param colony - Colony object with colonyid and name
+     */
+    addColony(colony: {
+        colonyid: string;
+        name: string;
+    }): Promise<any>;
+    /**
+     * Remove a colony (requires server private key)
+     * @param colonyName - Name of the colony to remove
+     */
+    removeColony(colonyName: string): Promise<any>;
+    getExecutors(colonyName: string): Promise<any>;
+    getExecutor(colonyName: string, executorName: string): Promise<any>;
+    addExecutor(executor: {
+        executorid: string;
+        executortype: string;
+        executorname: string;
+        colonyname: string;
+    }): Promise<any>;
+    approveExecutor(colonyName: string, executorName: string): Promise<any>;
+    removeExecutor(colonyName: string, executorName: string): Promise<any>;
+    submitFunctionSpec(spec: FunctionSpec): Promise<any>;
+    getProcess(processId: string): Promise<any>;
+    getProcesses(colonyName: string, count: number, state: ProcessState): Promise<any>;
+    removeProcess(processId: string): Promise<any>;
+    removeAllProcesses(colonyName: string, state?: number): Promise<any>;
+    assign(colonyName: string, timeout: number, executorPrvKey: string): Promise<any>;
+    closeProcess(processId: string, output: string[]): Promise<any>;
+    failProcess(processId: string, errors: string[]): Promise<any>;
+    submitWorkflowSpec(workflowSpec: {
+        colonyname: string;
+        functionspecs: FunctionSpec[];
+    }): Promise<any>;
+    getProcessGraph(processGraphId: string): Promise<any>;
+    getProcessGraphs(colonyName: string, count: number, state?: ProcessState): Promise<any>;
+    removeProcessGraph(processGraphId: string): Promise<any>;
+    removeAllProcessGraphs(colonyName: string, state?: ProcessState): Promise<any>;
+    addLog(processId: string, message: string, executorPrvKey: string): Promise<any>;
+    getLogs(colonyName: string, processId: string, executorName: string, count?: number, since?: number): Promise<any>;
+    getFunctions(executorName: string, colonyName: string): Promise<any>;
+    getCrons(colonyName: string, count?: number): Promise<any>;
+    getCron(cronId: string): Promise<any>;
+    addCron(cronSpec: any): Promise<any>;
+    removeCron(cronId: string): Promise<any>;
+    getGenerators(colonyName: string, count?: number): Promise<any>;
+    getGenerator(generatorId: string): Promise<any>;
+    addGenerator(generatorSpec: any): Promise<any>;
+    getUsers(colonyName: string): Promise<any>;
+    addUser(user: {
+        colonyname: string;
+        userid: string;
+        name: string;
+        email: string;
+        phone: string;
+    }): Promise<any>;
+    removeUser(colonyName: string, name: string): Promise<any>;
+    getFileLabels(colonyName: string, name?: string, exact?: boolean): Promise<any>;
+    getFiles(colonyName: string, label: string): Promise<any>;
+    addAttribute(attribute: {
+        targetid: string;
+        targetcolonyname: string;
+        targetprocessgraphid: string;
+        attributetype: number;
+        key: string;
+        value: string;
+    }): Promise<any>;
+    getAttribute(attributeId: string): Promise<any>;
+    /**
+     * Append a message to a process channel
+     * @param processId - ID of the process
+     * @param channelName - Name of the channel
+     * @param sequence - Client-assigned sequence number
+     * @param inReplyTo - Sequence number this message is replying to (0 if not a reply)
+     * @param payload - Message content (string or Uint8Array)
+     */
+    channelAppend(processId: string, channelName: string, sequence: number, inReplyTo: number, payload: string | Uint8Array): Promise<any>;
+    /**
+     * Read messages from a process channel
+     * @param processId - ID of the process
+     * @param channelName - Name of the channel
+     * @param afterSeq - Read messages after this sequence number (use 0 for all)
+     * @param limit - Maximum number of messages to return (0 for no limit)
+     */
+    channelRead(processId: string, channelName: string, afterSeq: number, limit: number): Promise<any[]>;
+    /**
+     * Subscribe to a channel using WebSocket for real-time updates
+     * @param processId - ID of the process
+     * @param channelName - Name of the channel
+     * @param afterSeq - Start reading after this sequence number
+     * @param timeout - Timeout in seconds for the subscription
+     * @param onMessage - Callback for new messages
+     * @param onError - Callback for errors
+     * @param onClose - Callback when connection closes
+     * @returns WebSocket instance for cleanup
+     */
+    subscribeChannel(processId: string, channelName: string, afterSeq: number, timeout: number, onMessage: (entries: any[]) => void, onError: (error: Error) => void, onClose: () => void): WebSocket;
+    /**
+     * Add a blueprint definition
+     * @param definition - Blueprint definition object
+     */
+    addBlueprintDefinition(definition: any): Promise<any>;
+    /**
+     * Get a blueprint definition by name
+     * @param colonyName - Name of the colony
+     * @param name - Name of the blueprint definition
+     */
+    getBlueprintDefinition(colonyName: string, name: string): Promise<any>;
+    /**
+     * Get all blueprint definitions in a colony
+     * @param colonyName - Name of the colony
+     */
+    getBlueprintDefinitions(colonyName: string): Promise<any[]>;
+    /**
+     * Remove a blueprint definition
+     * @param colonyName - Name of the colony (namespace)
+     * @param name - Name of the blueprint definition to remove
+     */
+    removeBlueprintDefinition(colonyName: string, name: string): Promise<void>;
+    /**
+     * Add a blueprint instance
+     * @param blueprint - Blueprint object
+     */
+    addBlueprint(blueprint: any): Promise<any>;
+    /**
+     * Get a blueprint by name
+     * @param colonyName - Name of the colony (namespace)
+     * @param name - Name of the blueprint
+     */
+    getBlueprint(colonyName: string, name: string): Promise<any>;
+    /**
+     * Get blueprints in a colony, optionally filtered by kind and location
+     * @param colonyName - Name of the colony (namespace)
+     * @param kind - Optional kind filter
+     * @param location - Optional location filter
+     */
+    getBlueprints(colonyName: string, kind?: string, location?: string): Promise<any[]>;
+    /**
+     * Update an existing blueprint
+     * @param blueprint - Updated blueprint object
+     * @param forceGeneration - Force generation bump even if spec unchanged
+     */
+    updateBlueprint(blueprint: any, forceGeneration?: boolean): Promise<any>;
+    /**
+     * Remove a blueprint
+     * @param colonyName - Name of the colony (namespace)
+     * @param name - Name of the blueprint to remove
+     */
+    removeBlueprint(colonyName: string, name: string): Promise<void>;
+    /**
+     * Update blueprint status (current state)
+     * @param colonyName - Name of the colony
+     * @param name - Name of the blueprint
+     * @param status - Status object representing current state
+     */
+    updateBlueprintStatus(colonyName: string, name: string, status: any): Promise<void>;
+    /**
+     * Trigger reconciliation for a blueprint
+     * @param colonyName - Name of the colony (namespace)
+     * @param name - Name of the blueprint
+     * @param force - Force reconciliation even if no changes detected
+     */
+    reconcileBlueprint(colonyName: string, name: string, force?: boolean): Promise<any>;
+    /**
+     * Subscribe to process state changes using WebSocket
+     * Use this to wait for a process to be assigned (RUNNING state) before subscribing to channels
+     * @param colonyName - Name of the colony
+     * @param processId - ID of the process to watch
+     * @param state - Target state to wait for (0=WAITING, 1=RUNNING, 2=SUCCESS, 3=FAILED)
+     * @param timeout - Timeout in seconds for the subscription
+     * @param onProcess - Callback when process reaches the target state
+     * @param onError - Callback for errors
+     * @param onClose - Callback when connection closes
+     * @returns WebSocket instance for cleanup
+     */
+    subscribeProcess(colonyName: string, processId: string, state: number, timeout: number, onProcess: (process: any) => void, onError: (error: Error) => void, onClose: () => void): WebSocket;
+}
+
+export { ColoniesClient, type ColoniesClientConfig, Crypto, type FunctionSpec, ProcessState, type RPCMessage, deriveId, generatePrivateKey, sign };