@keb-org/consol 0.0.0-stage → 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Kryat, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,266 @@
-# Temporary Holding Version
+# @keb-org/consol
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+TypeScript SDK for the consol memory engine, for three uses:
+
+1. **Local**: a vault on disk (or S3) in your own process. The SDK starts `consol stdio` as a child.
+2. **Hosted client**: the same `Vault` methods against a backend over HTTP.
+3. **Backend**: `Consol.handle` behind your own HTTP route, so clients of any language can use it.
+
+Node >= 22 or Bun, no dependencies; `import` and `require` share one implementation. The behaviour
+is the same in every mode.
+
+## The server binary and database
+
+The package does not build the server. The SDK uses the first of these that exists:
+
+1. the `binaryPath` option,
+2. the `CONSOL_BIN` environment variable,
+3. the platform package `@keb-org/consol-<platform>-<arch>`, resolved from the SDK's own location,
+4. `consol` (`consol.exe` on Windows) on `PATH`.
+
+A `binaryPath` or `CONSOL_BIN` that points nowhere fails at once with `binary_not_found` instead
+of falling through to the next place. Build the binary from the repository with
+`cargo build --release -p consol-cli`.
+
+The engine needs a database, PostgreSQL with pgvector or CockroachDB: pass `db` or set
+`CONSOL_DB_URL` in your process (the one `CONSOL_*` variable the SDK takes from its host).
+
+## 1. Local
+
+```ts
+import assert from 'node:assert/strict';
+import { createVault } from '@keb-org/consol';
+
+const vault = await createVault({ path: './memory', agent: 'assistant' });
+try {
+  await vault.remember({ id: 'deploy-day', content: 'Deploys run on Fridays.' });
+  const hits = await vault.recall({ query: 'deploys run' });
+  assert(hits.candidates.some((card) => card.source.id === 'deploy-day'));
+  const note = await vault.read({ id: 'deploy-day' });
+  console.log(note.text); // the complete Markdown file, frontmatter included
+} finally {
+  await vault.close();
+}
+```
+
+`createVault` fails with `vault_exists` when the directory already holds a vault, `openVault`
+(same options) with `vault_not_found` when it does not. Both start the server and check that its
+version matches the SDK's major.minor before they return.
+
+## 2. Hosted client
+
+```typescript
+import { openVault } from '@keb-org/consol';
+
+// A backend running `Consol.handle` (use 3) answers at `url`; `headers` are your auth with it.
+const vault = await openVault({
+  url: 'https://api.acme.dev/v1/consol',
+  headers: { Authorization: `Bearer ${process.env.ACME_KEY}` },
+  vault: 'demo', // the backend's name for the vault
+  agent: 'assistant',
+});
+try {
+  await vault.remember({ id: 'pref-1', content: 'The user prefers short answers.' });
+  const { candidates } = await vault.recall({ query: 'how long should answers be?' });
+  console.log(candidates.map((card) => card.source.id));
+} finally {
+  await vault.close();
+}
+```
+
+Backends create vaults, clients only open them: there is no `createVault` for a URL. Opening makes
+no request, so a missing vault is `vault_not_found` on the first call. Every method, error and
+deadline works as in the local use. A backend that refuses the vault or agent answers `forbidden`;
+a reply that is not a consol reply is `request_failed` (`status`; retryable on connect errors and
+502/503/504; a 401 or 403 is `forbidden`).
+
+## 3. Backend
+
+```typescript
+import express from 'express';
+import { Consol, httpStatus } from '@keb-org/consol';
+
+const consol = await Consol.start({ vaultRoot: './vaults' }); // or s3://bucket/prefix; db is $CONSOL_DB_URL
+await consol.createVault('acme'); // admin, once per tenant: `vault_exists` if it is there
+
+const app = express();
+app.use(express.json());
+app.post('/v1/consol', async (req, res) => {
+  const user = await authenticate(req); // YOUR auth: a session, a token, an API key
+  // ctx limits the request. It comes from your auth, NEVER from req.body or anything the caller sets.
+  const reply = await consol.handle(req.body, { vaults: [user.tenant], agent: user.id });
+  res.status(httpStatus(reply)).json(reply); // a tool error is a reply, not an exception
+});
+```
+
+- `handle(body, ctx)` takes `{ tool, vault?, agent?, args }` (the five tools), returns the reply
+  `{ version, text?, result?, error? }` as is, and is safe to call concurrently. `httpStatus` maps
+  it to 200, 400, 403, 404, 409, 413, 422, 503 or 500.
+- `ctx.vaults` lists the vaults the caller may use (`null` is any vault, on purpose; with one entry
+  the body may omit `vault`). `ctx.agent` pins the agent: a body naming another is `forbidden`.
+- `createVault(name)` / `openVault(name)` never go over the network (`vault_exists` /
+  `vault_not_found`). `Consol.start({ vault: path })` serves one vault, named `default`. Call
+  `close()` on shutdown. Engine options (`db`, `embedding`, ...) are listed below.
+
+## Methods
+
+Each method takes one object with the tool's wire argument names, plus optional
+`{ timeoutMs, signal }`, and returns the payload, not the wire reply.
+
+| Method | Arguments | Returns |
+|---|---|---|
+| `recall` | `{ query, include_history?, limit?, agent? }` | `RecallResult`: `text` is the progressive disclosure to give a model (cards by tier, intent block, hints); `candidates` are the same cards as data. Cards are leads: read a note before relying on it |
+| `read` | `{ id, continuation?, agent? }` | `ReadResult`; `text` is the exact file |
+| `remember` | `{ id, content?, expected_version?, summary?, entities?, facets?, scope?, team?, valid_from?, valid_to?, reflection?, agent? }` | `RememberResult` |
+| `remove` | `{ id, expected_version, reflection?, agent? }` | `RemoveResult`: the note is archived, its body stays readable |
+| `reflect` | one mode: `{}` discovery, `{ ids }`, `{ work_ids }` or `{ decisions }`; `limit?`, `agent?` | `ReflectResult`; delivered sources carry their `text` |
+| `callTool` | `(name, args)` with raw wire arguments | the raw `Reply`; tool errors come back as `reply.error` |
+| `close` | | final and idempotent |
+
+- `remember` without `expected_version` is create-only. To update, pass `source.content_version`
+  from your last `read`; a stale one fails with `version_conflict`. Omitted or `null` fields keep
+  their stored value, `""` or `[]` clears them.
+- `remember` and `remove` also take `{ resume: true, reflection, agent? }` to resume a pending
+  reflection-bound write. `reflection` is `{ work_id, basis_hash, action_key }`.
+- `byte_range` values are UTF-8 byte offsets: `sliceUtf8(text, start, end)` slices a JS string by them.
+- `vault.serverVersion` is the server version of the last reply. An agent exists from its first
+  `remember`; reading as an agent that wrote nothing finds nothing.
+
+## LLM tool loop
+
+```ts
+import { openVault } from '@keb-org/consol';
+
+const vault = await openVault({ path: './memory', agent: 'assistant' });
+const call = { name: 'recall', input: { query: 'deploys run' } }; // a tool call from your model
+
+const reply = await vault.callTool(call.name, call.input); // run it
+const toolResult = { content: JSON.stringify(reply), isError: reply.error !== undefined }; // hand it back
+
+console.log(toolResult.isError);
+await vault.close();
+```
+
+`callTool` never throws for a tool error, so the model sees the structured error and can fix its
+call. In the reply, `text` is guidance written for the model and `result` the payload, with note
+Markdown inline.
+
+The tool list is not on the wire. `consol schema` prints its JSON Schema: `oneOf` holds one
+envelope per tool, with the `args` schema and the description the model reads. An MCP client can
+use `consol mcp` instead.
+
+## Loading many notes
+
+Each `remember` is independent, so load a corpus with several writes in flight: with a hosted
+embedding provider most of a write is waiting on it, and concurrent writes overlap that wait
+(8 writers loaded 3.8x faster than one in a measured run).
+
+```typescript
+const queue = [...docs];                                      // docs: { id, content }[]
+await Promise.all(Array.from({ length: 8 }, async () => {
+  for (let doc; (doc = queue.shift()); ) await vault.remember(doc);
+}));
+```
+
+## Errors
+
+Every failure is a `ConsolError`. Branch on `code` and `retryable`, never on the message:
+
+```ts
+import { ConsolError, openVault } from '@keb-org/consol';
+
+const vault = await openVault({ path: './memory', agent: 'assistant' });
+try {
+  await vault.remember({ id: 'deploy-day', content: 'Deploys run on Mondays.' }); // create-only: the id exists
+} catch (error) {
+  if (!ConsolError.is(error) || error.code !== 'already_exists') throw error;
+  console.log(error.code, error.retryable, error.tool); // already_exists false remember
+} finally {
+  await vault.close();
+}
+```
+
+- `ConsolError.is(error)` also works across duplicate copies of the package, where `instanceof` fails.
+- `field` names the input at fault, relative to the call's arguments (`limit`,
+  `decisions[2].temporal`), `tool` the tool, and `result` keeps the raw `Reply` of a failed tool
+  call, including the durable state of a pending write (`error.result.result`).
+- Tool codes (contract section 4; accept unknown future ones): `invalid_argument`, `forbidden`,
+  `not_found`, `already_exists`, `version_conflict`, `team_not_attached`, `secret_rejected`,
+  `content_too_large`, `unsupported`, `work_not_found`, `basis_mismatch`, `already_decided`,
+  `unresolved_writes`, `postcondition_failed`, `index_unavailable`, `storage_unavailable`, `vault_busy`, `internal`.
+  Vault codes: `vault_exists`, `vault_not_found`.
+- SDK codes: `request_failed` (`status?`), `binary_not_found`, `incompatible_server` (server
+  major.minor differs from the SDK's), `server_exited`, `timeout`, `aborted`, `closed`.
+- `retryable: true` (`vault_busy`, `index_unavailable`, `storage_unavailable`, `request_failed` on a connect error or
+  502/503/504) means the identical request may succeed later. The SDK never retries by itself.
+
+## Lifecycle
+
+- `close()` is final and idempotent: in-flight and later calls fail with `closed`, and concurrent
+  calls share one completion. `await using vault = await openVault(...)` closes it for you.
+- An idle local vault does not keep Node or Bun alive: a script that forgets `close()` still
+  exits, and its server exits with it.
+- Every call has a deadline: 300 s by default, `timeoutMs` per vault or per call, `0` disables it.
+  `signal` aborts a call. Both only stop waiting: a write may still complete, so read the note
+  before you retry.
+- When a local server crashes, in-flight calls fail with `server_exited`. The next call starts the
+  server once more and opens the vault again (`vault_not_found` if it was deleted meanwhile).
+- Servers on one vault take turns on its lock, a lease in the database; a wait over 5 s fails with
+  `vault_busy`.
+
+## Configuration and logging
+
+Options of `createVault`, `openVault` (local) and `Consol.start` become the server's environment,
+never its argv. Inherited `CONSOL_*` and `RUST_LOG` variables are dropped first (except
+`CONSOL_DB_URL` as the default for `db`), so a stray host variable cannot change the agent or the
+vault.
+
+| Option | Meaning |
+|---|---|
+| `path` | local vault directory or `s3://bucket[/prefix]/<name]`, made absolute (`createVault`, `openVault`) |
+| `vaultRoot` / `vault` | `Consol.start`: a directory or `s3://` prefix of many vaults, or one vault |
+| `agent` | default agent of every call; a call's `agent` argument overrides it |
+| `db` | database URL (`CONSOL_DB_URL`): PostgreSQL with pgvector, or CockroachDB |
+| `embedding` | `{ url, model, apiKey?, dims?, queryPrefix?, batchSize?, timeoutMs? }` (`CONSOL_EMBED_*`); without it recall is lexical-only |
+| `jev` | `{ url?, apiKey?, model?, auto? }` (`CONSOL_JEV_*`); experimental advice |
+| `env` | any other server variable, applied last |
+| `binaryPath` | server binary, see above |
+| `timeoutMs` | default per-call deadline in ms |
+| `onLog` | `(line) => void`, receives each server stderr line |
+
+The server's stderr is captured and never written to your process's stderr. Pass `onLog` to see
+it, and `env: { RUST_LOG: 'debug' }` for more detail. A `server_exited` error carries the server's
+last lines in its message and in `stderrTail`.
+
+## Types and stability
+
+Exports: `createVault`, `openVault`, `Vault`, `Consol`, `httpStatus`, `ConsolError`, `sliceUtf8` and
+the payload and argument types (`Reply`, `Context`, `RecallResult`, `ReadResult`, `RememberResult`,
+`RemoveResult`, `ReflectResult`, `Decision`, `ToolError`, ...). Payloads may gain fields in any
+release. Experimental, so they may change in a minor release: hub and promotion work, leads and
+decisions, and every `jev`, `jev_triage` and `jev_advisory` field (typed as plain records).
+
+## Development
+
+```bash
+bun install
+bun run build       # dist/index.js (CJS), dist/index.mjs (ESM wrapper), dist/*.d.ts
+bun run typecheck   # tsc over src
+# the rest needs the real binary and a database:
+export CONSOL_BIN=/abs/consol CONSOL_DB_URL=postgres://...
+bun run test            # pure checks, hosted-client stub, real-binary suite on Node, strict tsc incl. conformance
+bun run test:consumers  # packed tarballs in Bun, Node ESM, Node CJS and platform-package projects
+node test/capture.mjs   # recapture test/fixtures from the real server (without CONSOL_BIN it only regenerates the types check)
+```
+
+`bun run test`, not `bun test` (that is Bun's own runner). Consumer logs go to
+`consumers/<project>.log`.
+
+## Publishing platform binaries
+
+1. `bun run build`
+2. `bun scripts/make-platform-package.mjs <binary> <platform> <arch> <outDir>` per target
+   (win32-x64, darwin-arm64, darwin-x64, linux-x64, linux-arm64); the binary is stored as
+   `bin/consol[.exe]` whatever its file name
+3. Publish each platform package, then `bun scripts/inject-optional-deps.mjs <publish-copy-of-package.json>`
+4. `bun publish` the main package (`publishConfig.access` is `public`)

package/dist/consol.d.ts

@@ -0,0 +1,36 @@
+import { type EngineOptions } from "./stdio";
+import type { CallOptions, Context } from "./transport";
+import type { Reply } from "./types";
+/** Exactly one of `vaultRoot` / `vault`, plus the engine settings (`db`, `embedding`, `jev`, ...). */
+export interface ConsolOptions extends EngineOptions {
+    /** Directory or `s3://bucket[/prefix]` holding the vaults: vault `x` is `<root>/x`. */
+    vaultRoot?: string;
+    /** One vault (directory or `s3://bucket[/prefix]/<name>`), named `default`. */
+    vault?: string;
+}
+export declare function httpStatus(reply: Reply): number;
+/**
+ * The engine in your own backend: one `consol stdio` child behind `handle`. A backend authorizes
+ * the request itself, then passes the result as `ctx`.
+ *
+ * SECURITY: `ctx` (the vaults the caller may use, the agent they are) must come from YOUR auth,
+ * for example the session of the signed-in user. NEVER build it from the request body, query or
+ * headers: the body is untrusted input, `ctx` is what limits it.
+ */
+export declare class Consol {
+    private readonly transport;
+    private constructor();
+    static start(options: ConsolOptions): Promise<Consol>;
+    /**
+     * Runs one envelope (`{tool, vault?, agent?, args}`, usually `req.body`) under `ctx` and returns
+     * the reply as is. A tool error is a reply with `error`, not an exception: answer with
+     * `res.status(httpStatus(reply)).json(reply)`. Safe to call concurrently.
+     */
+    handle(body: unknown, ctx: Context, options?: CallOptions): Promise<Reply>;
+    /** Rejects with `vault_exists` when the vault is there. A vault is created here, never over the wire. */
+    createVault(name: string, options?: CallOptions): Promise<void>;
+    /** Rejects with `vault_not_found` when there is no such vault. */
+    openVault(name: string, options?: CallOptions): Promise<void>;
+    /** Final and idempotent: in-flight and later calls reject with `closed`. */
+    close(): Promise<void>;
+}

package/dist/errors.d.ts

@@ -0,0 +1,29 @@
+import type { Reply } from "./types";
+/** Branchable error codes: wire tool codes (contract section 4) plus the SDK codes of section 6. */
+export type ConsolErrorCode = "invalid_argument" | "not_found" | "already_exists" | "version_conflict" | "team_not_attached" | "secret_rejected" | "content_too_large" | "unsupported" | "work_not_found" | "basis_mismatch" | "already_decided" | "unresolved_writes" | "postcondition_failed" | "index_unavailable" | "storage_unavailable" | "vault_busy" | "forbidden" | "vault_exists" | "vault_not_found" | "internal" | "request_failed" | "binary_not_found" | "incompatible_server" | "server_exited" | "timeout" | "aborted" | "closed" | (string & {});
+export interface ConsolErrorDetails {
+    retryable?: boolean;
+    field?: string;
+    tool?: string;
+    /** HTTP status of a `request_failed` / `forbidden` that came from the transport, not the engine. */
+    status?: number;
+    stderrTail?: string;
+    result?: Reply;
+    cause?: unknown;
+}
+/** One error class; branch on `code`, never on the message. */
+export declare class ConsolError extends Error {
+    readonly code: ConsolErrorCode;
+    readonly retryable: boolean;
+    readonly field?: string;
+    readonly tool?: string;
+    readonly status?: number;
+    readonly stderrTail?: string;
+    /** Raw reply of a failed typed call: durable write state (`result.result`) survives the throw. */
+    readonly result?: Reply;
+    constructor(code: ConsolErrorCode, message: string, details?: ConsolErrorDetails);
+    /** instanceof breaks across duplicate package copies; the string brand does not. */
+    static is(error: unknown): error is ConsolError;
+}
+/** The ConsolError for an error reply, or undefined for a result reply. */
+export declare function replyError(reply: Reply, tool?: string): ConsolError | undefined;

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+export { ConsolError, type ConsolErrorCode } from "./errors";
+export { Consol, httpStatus, type ConsolOptions } from "./consol";
+export type { CallOptions, Context } from "./transport";
+export type { EmbeddingOptions, EngineOptions, JevOptions } from "./stdio";
+export { Vault, createVault, openVault, sliceUtf8, type VaultOptions, type HostedOptions, type RecallArgs, type ReadArgs, type RememberArgs, type RemoveArgs, type ResumeArgs, type ReflectArgs, type ReflectionContext, type Decision, type InspectedSource, type InspectedSpan, type TemporalAssessment, type PairAssertion, } from "./vault";
+export type { Reply, ToolError, Unavailable, Status, ByteRange, Scope, SourceRef, RelationshipKind, Relationship, RecallRetrieval, SourceExcerpt, RecallCard, RecallResult, ReadResult, RememberResult, RemoveResult, ReflectResult, AffectedDoc, EntityAffectedGroup, WriteState, WriteFailureCode, WriteReceipt, WriteResolution, WriteOutcome, ReflectionWork, SourceWork, PairWork, HubWork, PromotionWork, DeliveredSource, CompleteDelivery, FailedDelivery, ReflectDelivery, DecisionReceipt, SourceDisposition, PairDisposition, } from "./types";