@sema-agent/connectors 0.1.0 → 0.3.0

package/dist/launcher.js

@@ -1,19 +1,21 @@
 import { spawn } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
-import { readFileSync } from 'node:fs';
-import { createRequire } from 'node:module';
 import { isIPv4, isIPv6 } from 'node:net';
-import { dirname, join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
 import { absoluteDir, given } from './config-dir.js';
 import { CONNECTOR_NAME } from './connector-name.js';
 import { serveHttp } from './http.js';
 import { keySetSource, obtainKeySet } from './jwks.js';
 import { hostedResolver, standaloneResolver } from './principal.js';
 import { Scrubber, secretMatcher } from './redact.js';
-import { copyForLaunch, openFileSecrets } from './secrets.js';
+import { resolveConnector } from './resolve-connector.js';
+import { NO_HANDLES, copyForLaunch, handleDirectory } from './secrets.js';
+import { seed } from './seed.js';
 import { INSPECT_LIMIT_BYTES, Session } from './session.js';
 import { mintStatus } from './status.js';
-const EXIT = { ok: 0, guard: 2, notInstalled: 4, connectorFailed: 5 };
+import { writeFailure } from './write-all.js';
+const EXIT = { ok: 0, notSeeded: 1, guard: 2, notInstalled: 4, connectorFailed: 5 };
 const NOT_INSTALLED = 'connector.not_installed';
 const INTERNAL = 'connector.internal';
 const STOP_SIGNALS = ['SIGTERM', 'SIGINT', 'SIGHUP'];
@@ -21,20 +23,15 @@ const TEARDOWN_STEP_MS = 1000;
 const STALL_STEPS = 10;
 const STALL_MS = STALL_STEPS * TEARDOWN_STEP_MS;
 const PIECE_BYTES = 1024;
-const USAGE = 'usage: sema-connector run <name> --mode standalone|hosted --transport stdio|http [--listen <ip>:<port>] [--config-dir <absolute path>]';
+const USAGE = 'usage: sema-connector run <name> --mode standalone|hosted --transport stdio|http [--listen <ip>:<port>] [--config-dir <absolute path>]\n       sema-connector seed --config-dir <absolute path>';
 const FLAGS = ['--mode', '--transport', '--listen', '--config-dir'];
+const SEED_FLAGS = ['--config-dir'];
 const CHILD_ENV = ['HOME', 'LOGNAME', 'USER', 'PATH', 'SHELL', 'TERM', 'LANG', 'LC_ALL', 'TZ'];
-function guard(argv, env) {
-    if (argv[0] !== 'run')
-        return 'expected the command "run"';
-    const name = argv[1];
-    if (name === undefined || name.startsWith('-'))
-        return 'missing the connector <name> after "run"';
-    if (!CONNECTOR_NAME.test(name))
-        return '<name> must be lowercase letters and digits in hyphen-separated words (it names the package @sema-agent/connector-<name>)';
+const CONNECTOR_DIR = 'CONNECTOR_DIR';
+function flagValues(argv, from, allowed) {
     const values = new Map();
-    for (let i = 2; i < argv.length; i += 2) {
-        const flag = FLAGS.find((f) => f === argv[i]);
+    for (let i = from; i < argv.length; i += 2) {
+        const flag = allowed.find((f) => f === argv[i]);
         if (flag === undefined)
             return `unknown argument at position ${i + 1}`;
         if (values.has(flag))
@@ -44,6 +41,31 @@ function guard(argv, env) {
             return `${flag} needs a value`;
         values.set(flag, value);
     }
+    return values;
+}
+function guard(argv, env) {
+    if (argv[0] === 'seed') {
+        const values = flagValues(argv, 1, SEED_FLAGS);
+        if (typeof values === 'string')
+            return values;
+        const configDirArgument = values.get('--config-dir');
+        if (configDirArgument === undefined)
+            return 'seed requires --config-dir <absolute path>, the configuration directory it seeds (there is no default)';
+        const configDir = absoluteDir(configDirArgument);
+        if (configDir === null)
+            return '--config-dir must be an absolute path';
+        return { command: 'seed', configDir };
+    }
+    if (argv[0] !== 'run')
+        return 'expected the command "run" or "seed"';
+    const name = argv[1];
+    if (name === undefined || name.startsWith('-'))
+        return 'missing the connector <name> after "run"';
+    if (!CONNECTOR_NAME.test(name))
+        return '<name> must be lowercase letters and digits in hyphen-separated words (it names the package @sema-agent/connector-<name>)';
+    const values = flagValues(argv, 2, FLAGS);
+    if (typeof values === 'string')
+        return values;
     const mode = values.get('--mode');
     const transport = values.get('--transport');
     const listen = values.get('--listen');
@@ -66,7 +88,7 @@ function guard(argv, env) {
         const configDir = absoluteDir(configDirArgument);
         if (configDir === null)
             return '--config-dir must be an absolute path';
-        return { name, mode, configDir };
+        return { command: 'run', name, mode, configDir };
     }
     if (transport !== 'http')
         return 'hosted mode requires --transport http';
@@ -86,7 +108,7 @@ function guard(argv, env) {
     const keySet = keySetSource(env, issuer);
     if (typeof keySet === 'string')
         return keySet;
-    return { name, mode, listen: address, issuer, audience, keySet };
+    return { command: 'run', name, mode, listen: address, issuer, audience, keySet };
 }
 function listenAddress(value) {
     const m = /^(?:\[([^\]]+)\]|([^:[\]]+)):([1-9][0-9]{0,4})$/.exec(value);
@@ -97,31 +119,6 @@ function listenAddress(value) {
     const port = Number(m[3]);
     return hostOk && port <= 65535 ? { host, port } : null;
 }
-function resolveConnector(name) {
-    const packageName = `@sema-agent/connector-${name}`;
-    let manifestPath;
-    try {
-        manifestPath = createRequire(new URL('../package.json', import.meta.url)).resolve(`${packageName}/package.json`);
-    }
-    catch {
-        return { packageName, reason: 'is not installed where the launcher can resolve it' };
-    }
-    let bin;
-    try {
-        bin = JSON.parse(readFileSync(manifestPath, 'utf8')).bin;
-    }
-    catch {
-        return { packageName, reason: 'has a package.json that cannot be read' };
-    }
-    let rel;
-    if (typeof bin === 'string')
-        rel = bin;
-    else if (bin !== null && typeof bin === 'object' && !Array.isArray(bin) && Object.keys(bin).length === 1)
-        rel = Object.values(bin)[0];
-    if (typeof rel !== 'string' || rel === '')
-        return { packageName, reason: 'does not declare exactly one bin' };
-    return { packageName, binPath: join(dirname(manifestPath), rel) };
-}
 class Out {
     stream;
     held = [];
@@ -369,17 +366,19 @@ function pump(src, dst, framing, also = [], passing, giveUp) {
         src.on('close', finish);
     });
 }
-function childEnvironment(env, secrets) {
+function childEnvironment(env, connectorDir, secrets) {
     const out = {};
     for (const name of CHILD_ENV) {
         const value = env[name];
         if (value !== undefined)
             out[name] = value;
     }
+    out[CONNECTOR_DIR] = connectorDir;
     for (const [name, path] of secrets.files)
         out[`${name}_FILE`] = path;
     return out;
 }
+const hostedConnectorDir = (name) => join(tmpdir(), `sema-connector-${randomUUID()}`, 'connectors', name);
 const describe = (o) => `connector process ${o.stalled ? `stopped reading its input (stalled input: none of it taken for ${STALL_MS / 1000} s) and ` : ''}${o.error ? `could not be started (${o.error.message})` : o.signal !== null ? `was terminated by signal ${o.signal}` : `exited with code ${String(o.code)}`}`;
 const abnormal = (o) => o.code !== 0 || o.stalled === true;
 function startConnector(binPath, childEnv, session, scrubber, toUpstream, upstreamOut, log, stderrAfterExitMs) {
@@ -491,15 +490,14 @@ async function runStdio(packageName, binPath, childEnv, session, scrubber, stdou
     return EXIT.connectorFailed;
 }
 async function runHosted(plan, keys, packageName, binPath, env, log) {
-    const childEnv = childEnvironment(env, NO_SECRETS);
     const start = (deliver) => {
         const session = new Session({
             connector: plan.name,
-            status: (tools) => mintStatus({ handles: [], nowMs: Date.now(), tools }),
+            status: (tools) => mintStatus({ handles: NO_HANDLES.handles, nowMs: Date.now(), tools }),
             ownIdPrefix: `sema-connector/${randomUUID()}`,
-            withheld: secretMatcher(NO_SECRETS.values),
+            withheld: secretMatcher(NO_HANDLES.values),
         });
-        const run = startConnector(binPath, childEnv, session, new Scrubber(NO_SECRETS.values), (data) => deliver(String(data)), null, log, TEARDOWN_STEP_MS);
+        const run = startConnector(binPath, childEnvironment(env, hostedConnectorDir(plan.name), NO_HANDLES), session, new Scrubber(NO_HANDLES.values), (data) => deliver(String(data)), null, log, TEARDOWN_STEP_MS);
         return {
             send: (raw, principal) => run.fromUpstream(raw, principal),
             busy: () => run.toChild.behind(),
@@ -524,7 +522,24 @@ async function runHosted(plan, keys, packageName, binPath, env, log) {
     setTimeout(abandon, TEARDOWN_STEP_MS);
     return EXIT.ok;
 }
-const NO_SECRETS = { files: new Map(), values: [], remove: () => undefined };
+const readingLine = (r) => `seed: ${r.name} ${r.outcome}${r.outcome === 'not_installed' ? ` (${r.message})` : r.outcome === 'refused' ? ` (${r.reason}: ${r.message})` : ''}\n`;
+function runSeed(configDir, stdout, log) {
+    let result;
+    try {
+        result = seed({ configDir });
+    }
+    catch (e) {
+        log.line(`sema-connector: seed: failed (${writeFailure(e)}); the marker was not written\n`);
+        return EXIT.notSeeded;
+    }
+    for (const r of result.readings)
+        stdout.write(readingLine(r));
+    if (!result.ok) {
+        log.line(`sema-connector: seed: ${result.reason}: ${result.message}\n`);
+        return EXIT.notSeeded;
+    }
+    return result.readings.every((r) => r.outcome === 'mounted' || r.outcome === 'present' || r.outcome === 'not_installed') ? EXIT.ok : EXIT.notSeeded;
+}
 export async function main(argv, env) {
     const stdout = new Out(process.stdout);
     const log = new Log(new Out(process.stderr));
@@ -537,6 +552,8 @@ export async function main(argv, env) {
         log.line(`sema-connector: ${plan}\n${USAGE}\n`);
         return finish(EXIT.guard);
     }
+    if (plan.command === 'seed')
+        return finish(runSeed(plan.configDir, stdout, log));
     if (plan.mode === 'hosted') {
         const keys = await obtainKeySet(plan.keySet);
         if (!keys.ok) {
@@ -550,20 +567,18 @@ export async function main(argv, env) {
         }
         return finish(await runHosted(plan, keys.keys, resolved.packageName, resolved.binPath, env, log));
     }
-    const listing = openFileSecrets({ configDir: plan.configDir, connector: plan.name }).list();
-    if (!listing.ok) {
-        log.line(`sema-connector: ${INTERNAL}: ${listing.problem}\n`);
-        return finish(EXIT.guard);
-    }
-    const { handles } = listing;
     let secrets;
     try {
-        secrets = copyForLaunch(plan.configDir, plan.name, handles);
+        secrets = copyForLaunch(plan.configDir, plan.name);
     }
     catch {
         log.line(`sema-connector: ${INTERNAL}: connector ${plan.name}: its secrets cannot be read for this launch\n`);
         return finish(EXIT.guard);
     }
+    if (!secrets.ok) {
+        log.line(`sema-connector: ${INTERNAL}: ${secrets.problem}\n`);
+        return finish(EXIT.guard);
+    }
     const resolved = resolveConnector(plan.name);
     if ('reason' in resolved) {
         secrets.remove();
@@ -572,12 +587,12 @@ export async function main(argv, env) {
     }
     const session = new Session({
         connector: plan.name,
-        status: (tools) => mintStatus({ handles, nowMs: Date.now(), tools }),
+        status: (tools) => mintStatus({ handles: secrets.handles, nowMs: Date.now(), tools }),
         ownIdPrefix: `sema-connector/${randomUUID()}`,
         withheld: secretMatcher(secrets.values),
     });
     try {
-        return finish(await runStdio(resolved.packageName, resolved.binPath, childEnvironment(env, secrets), session, new Scrubber(secrets.values), stdout, log));
+        return finish(await runStdio(resolved.packageName, resolved.binPath, childEnvironment(env, handleDirectory(plan.configDir, plan.name), secrets), session, new Scrubber(secrets.values), stdout, log));
     }
     finally {
         secrets.remove();

package/dist/lock-file.d.ts

@@ -0,0 +1 @@
+export declare function takeLockFile(path: string): (() => void) | null;

package/dist/lock-file.js

@@ -0,0 +1,34 @@
+import { randomBytes } from 'node:crypto';
+import { closeSync, linkSync, openSync, readFileSync, rmSync } from 'node:fs';
+import { writeAll } from './write-all.js';
+export function takeLockFile(path) {
+    const token = `${process.pid} ${randomBytes(12).toString('hex')}`;
+    const tmp = `${path}.${randomBytes(6).toString('hex')}`;
+    const fd = openSync(tmp, 'wx', 0o600);
+    try {
+        try {
+            writeAll(fd, token);
+        }
+        finally {
+            closeSync(fd);
+        }
+        linkSync(tmp, path);
+    }
+    catch (e) {
+        if (e.code === 'EEXIST')
+            return null;
+        throw e;
+    }
+    finally {
+        rmSync(tmp, { force: true });
+    }
+    return () => {
+        let held = '';
+        try {
+            held = readFileSync(path, 'utf8');
+        }
+        catch { }
+        if (held === token)
+            rmSync(path, { force: true });
+    };
+}

package/dist/mount.d.ts

@@ -2,7 +2,7 @@ export interface MountOptions {
     readonly name: string;
     readonly configDir: string;
 }
-export type MountRefusal = 'config_dir' | 'invalid_name' | 'reserved_name' | 'unreadable' | 'unexpected_shape' | 'conflict' | 'locked' | 'changed_during_write';
+export type MountRefusal = 'config_dir' | 'invalid_name' | 'reserved_name' | 'unreadable' | 'unexpected_shape' | 'conflict' | 'locked' | 'changed_during_write' | 'write_failed';
 export type MountResult = {
     readonly ok: true;
     readonly changed: boolean;
@@ -12,6 +12,10 @@ export type MountResult = {
     readonly reason: MountRefusal;
     readonly message: string;
 };
+export declare function nameRefusal(name: unknown): {
+    readonly reason: 'invalid_name' | 'reserved_name';
+    readonly message: string;
+} | null;
 export declare function standaloneEntry(name: string, configDir: string): {
     name: string;
     transport: {

package/dist/mount.js

@@ -1,10 +1,19 @@
-import { closeSync, fchmodSync, fsyncSync, linkSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeFileSync, writeSync } from 'node:fs';
+import { closeSync, fchmodSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync } from 'node:fs';
 import { randomBytes } from 'node:crypto';
 import { dirname, join } from 'node:path';
 import { absoluteDir } from './config-dir.js';
 import { CONNECTOR_NAME } from './connector-name.js';
 import { skipWhitespace, spansOf } from './json-text.js';
+import { takeLockFile } from './lock-file.js';
+import { writeAll, writeFailure } from './write-all.js';
 const RESERVED_NAMES = ['sema-connector'];
+export function nameRefusal(name) {
+    if (typeof name !== 'string' || !CONNECTOR_NAME.test(name))
+        return { reason: 'invalid_name', message: 'a connector name is lowercase letters and digits in hyphen-separated words' };
+    if (RESERVED_NAMES.includes(name))
+        return { reason: 'reserved_name', message: `${name} is a reserved name` };
+    return null;
+}
 export function standaloneEntry(name, configDir) {
     return { name, transport: { kind: 'stdio', command: 'sema-connector', args: ['run', name, '--mode', 'standalone', '--transport', 'stdio', '--config-dir', configDir] } };
 }
@@ -69,32 +78,6 @@ function plan(before, entry) {
     return { text };
 }
 const LOCK_FILE = '.mcp.json.lock';
-function lock(dir) {
-    const path = join(dir, LOCK_FILE);
-    const token = `${process.pid} ${randomBytes(12).toString('hex')}`;
-    const tmp = join(dir, `${LOCK_FILE}.${randomBytes(6).toString('hex')}`);
-    writeFileSync(tmp, token, { mode: 0o600, flag: 'wx' });
-    try {
-        linkSync(tmp, path);
-    }
-    catch (e) {
-        if (e.code === 'EEXIST')
-            return null;
-        throw e;
-    }
-    finally {
-        rmSync(tmp, { force: true });
-    }
-    return () => {
-        let held = '';
-        try {
-            held = readFileSync(path, 'utf8');
-        }
-        catch { }
-        if (held === token)
-            rmSync(path, { force: true });
-    };
-}
 const readOrNull = (file) => {
     try {
         return readFileSync(file, 'utf8');
@@ -108,17 +91,22 @@ const readOrNull = (file) => {
 export function mount(options) {
     const refuse = (reason, message) => ({ ok: false, reason, message });
     const name = options?.name;
-    if (typeof name !== 'string' || !CONNECTOR_NAME.test(name))
-        return refuse('invalid_name', 'a connector name is lowercase letters and digits in hyphen-separated words');
-    if (RESERVED_NAMES.includes(name))
-        return refuse('reserved_name', `${name} is a reserved name`);
+    const named = nameRefusal(name);
+    if (named !== null)
+        return refuse(named.reason, named.message);
     const configDir = absoluteDir(options.configDir);
     if (configDir === null)
         return refuse('config_dir', 'configDir must be the configuration directory as an absolute path; mount reads no environment variable and has no default');
     const dir = join(configDir, 'config.d');
     const file = join(dir, 'mcp.json');
-    mkdirSync(dir, { recursive: true });
-    const unlock = lock(dir);
+    let unlock;
+    try {
+        mkdirSync(dir, { recursive: true });
+        unlock = takeLockFile(join(dir, LOCK_FILE));
+    }
+    catch (e) {
+        return refuse('write_failed', `config.d or its lock could not be written (${writeFailure(e)}); config.d/mcp.json was left as it was`);
+    }
     if (unlock === null)
         return refuse('locked', `another mount is writing config.d/mcp.json; try again, and if no mount is running remove config.d/${LOCK_FILE}`);
     try {
@@ -155,8 +143,7 @@ function write(file, entry, refuse) {
         try {
             if (mode !== null)
                 fchmodSync(fd, mode);
-            writeSync(fd, text);
-            fsyncSync(fd);
+            writeAll(fd, text);
         }
         finally {
             closeSync(fd);
@@ -169,7 +156,7 @@ function write(file, entry, refuse) {
     }
     catch (e) {
         rmSync(tmp, { force: true });
-        throw e;
+        return refuse('write_failed', `config.d/mcp.json could not be written (${writeFailure(e)}); it was left as it was and nothing else was written`);
     }
     return { ok: true, changed: true, file };
 }

package/dist/pinned-upstream.d.ts

@@ -0,0 +1,21 @@
+import { type Env } from './config-dir.js';
+export interface PinnedUpstreamOptions {
+    readonly connector: string;
+    readonly upstream: string;
+    readonly lockFile: URL;
+    readonly handles: readonly string[];
+}
+export type UpstreamStart = {
+    readonly ok: true;
+    readonly connectorDir: string;
+    readonly files: Readonly<Record<string, string>>;
+    run(args: readonly string[], extra: Readonly<Record<string, string>>): Promise<number>;
+} | {
+    readonly ok: false;
+    readonly problem: string;
+};
+export interface PinnedUpstream {
+    install(configDir: string): Promise<number>;
+    start(env: Env): UpstreamStart;
+}
+export declare function pinnedUpstream(options: PinnedUpstreamOptions): PinnedUpstream;