dexterm-cli 0.1.7 → 0.2.1

package/dist/packages/autotrade/src/outbox.js

@@ -0,0 +1,394 @@
+// dexterm autotrade — the trade-fire outbox (task-1645, 0.2.0 card A).
+//
+// Every real-mode signal is written here (fsync) BEFORE the runner books its fill, then delivered to each channel
+// the user enabled: at-least-once, in nonce order per channel, retried with backoff (1 s doubling to 30 s) until
+// the channel acks, rejects it for good, or the signal's TTL passes (then it is marked expired and never sent late).
+//
+//   $DEXTERM_HOME/autotrade/real/outbox.jsonl   append-only records, replayed on open (a restart resumes delivery)
+//       {k:"enq", signal, channels}             a signal and the channels it was addressed to
+//       {k:"ack"|"rej"|"exp"|"drop", nonce, ch} a channel's terminal outcome (drop: the channel is no longer set up)
+//       {k:"try", nonce, ch, why, attempt}      a failed attempt (retried)
+//       {k:"run", runId, state:"open"|"closed"} run markers: a run left "open" died without its halted signal
+//       {k:"base", lastNonce}                   the head of an outbox started by compaction (r2): the nonce floor
+//   $DEXTERM_HOME/autotrade/real/nonce.json     the last nonce handed out (written before the signal exists)
+//   $DEXTERM_HOME/autotrade/real/outbox.lock    one writer at a time (a stale lock of a dead pid is reclaimed)
+//
+// Nonces only grow: the next one is max(nonce.json, the outbox's base record, the newest in the outbox) + 1, so
+// neither a restart, a compaction nor a lost nonce file can hand out a number twice. A record that cannot be parsed is skipped and counted (`corrupt`).
+import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeSync } from 'node:fs';
+import { join } from 'node:path';
+import { writeJsonAtomic } from "./ledger.js";
+import { validateSignal } from "./signal.js";
+export const CHANNEL_NAMES = ['webhook', 'cmd', 'jsonl', 'mcp', 'telegram'];
+/** Channels an executor can read. The Telegram copy is for the human: a bot cannot read another bot's messages. */
+export const EXECUTOR_CHANNELS = ['webhook', 'cmd', 'jsonl', 'mcp'];
+export const realDir = (home) => join(home, 'autotrade', 'real');
+export const outboxPath = (home) => join(realDir(home), 'outbox.jsonl');
+export const noncePath = (home) => join(realDir(home), 'nonce.json');
+const lockPath = (home) => join(realDir(home), 'outbox.lock');
+export const BACKOFF_MIN_MS = 1000;
+export const BACKOFF_MAX_MS = 30_000;
+const COMPACT_ABOVE_BYTES = 1 << 20;
+const MAX_PER_PUMP = 50;
+const alive = (pid) => { try {
+    process.kill(pid, 0);
+    return true;
+}
+catch (e) {
+    return e.code === 'EPERM';
+} };
+function appendSync(path, line) {
+    const fd = openSync(path, 'a', 0o600);
+    try {
+        writeSync(fd, line.endsWith('\n') ? line : `${line}\n`);
+        fsyncSync(fd);
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+export class Outbox {
+    home;
+    path;
+    channels = new Map();
+    defer;
+    entries = [];
+    byNonce = new Map();
+    runs = new Map();
+    health = new Map();
+    now;
+    lastNonce = 0;
+    running = null;
+    queued = null;
+    locked = false;
+    corrupt = 0;
+    constructor(home, o) {
+        this.home = home;
+        this.path = outboxPath(home);
+        this.now = o.now ?? Date.now;
+        for (const c of o.channels)
+            this.channels.set(c.name, c);
+        this.defer = new Set(o.defer ?? []);
+    }
+    /** Open (and replay) the outbox of a dexterm home. Throws when another live process holds it. */
+    static open(home, o) {
+        mkdirSync(realDir(home), { recursive: true, mode: 0o700 });
+        const b = new Outbox(home, o);
+        if (!o.noLock)
+            b.lock();
+        b.replay();
+        // Only the lock holder compacts (it writes): a read-only view never moves the file under a running writer.
+        if (!o.noLock)
+            b.compact();
+        return b;
+    }
+    lock() {
+        const p = lockPath(this.home);
+        for (let i = 0; i < 2; i++) {
+            try {
+                const fd = openSync(p, 'wx', 0o600);
+                writeSync(fd, `${JSON.stringify({ pid: process.pid, at: this.now() })}\n`);
+                closeSync(fd);
+                this.locked = true;
+                return;
+            }
+            catch (e) {
+                if (e.code !== 'EEXIST')
+                    throw e;
+            }
+            let pid = 0;
+            try {
+                pid = Number(JSON.parse(readFileSync(p, 'utf8')).pid) || 0;
+            }
+            catch {
+                pid = 0;
+            }
+            if (pid && pid !== process.pid && alive(pid))
+                throw new Error(`the trade-fire outbox is in use by pid ${pid} (a running real-mode runner); stop it first`);
+            rmSync(p, { force: true });
+        }
+        throw new Error('could not lock the trade-fire outbox');
+    }
+    close() {
+        if (!this.locked)
+            return;
+        try {
+            const d = JSON.parse(readFileSync(lockPath(this.home), 'utf8'));
+            if (d.pid === process.pid)
+                rmSync(lockPath(this.home), { force: true });
+        }
+        catch { /* gone */ }
+        this.locked = false;
+    }
+    /** Swap the channel set (config changed). Pending deliveries to a channel that is gone are dropped at the next pump. */
+    setChannels(cs) { this.channels = new Map(cs.map((c) => [c.name, c])); }
+    channelNames() { return [...this.channels.keys()]; }
+    replay() {
+        if (existsSync(noncePath(this.home))) {
+            try {
+                const n = Number(JSON.parse(readFileSync(noncePath(this.home), 'utf8')).last);
+                if (Number.isSafeInteger(n) && n > 0)
+                    this.lastNonce = n;
+            }
+            catch {
+                this.corrupt++;
+            }
+        }
+        if (!existsSync(this.path))
+            return;
+        for (const raw of readFileSync(this.path, 'utf8').split('\n')) {
+            if (!raw.trim())
+                continue;
+            let r;
+            try {
+                r = JSON.parse(raw);
+            }
+            catch {
+                this.corrupt++;
+                continue;
+            }
+            this.apply(r, false);
+        }
+    }
+    /** Apply one record to the in-memory state (replay and live writes share this). */
+    apply(r, live) {
+        const at = typeof r.at === 'number' ? r.at : 0;
+        if (r.k === 'enq') {
+            const v = validateSignal(r.signal);
+            if (!v.ok || this.byNonce.has(v.signal.nonce)) {
+                this.corrupt++;
+                return;
+            }
+            const chs = Array.isArray(r.channels) ? r.channels.filter((c) => CHANNEL_NAMES.includes(c)) : [];
+            const e = { signal: v.signal, line: JSON.stringify(v.signal), ch: new Map(chs.map((c) => [c, { done: null, attempts: 0, nextAt: 0, lastWhy: null }])) };
+            this.entries.push(e);
+            this.byNonce.set(e.signal.nonce, e);
+            this.lastNonce = Math.max(this.lastNonce, e.signal.nonce);
+            return;
+        }
+        if (r.k === 'base') {
+            const n = Number(r.lastNonce);
+            if (Number.isSafeInteger(n) && n > 0)
+                this.lastNonce = Math.max(this.lastNonce, n);
+            else
+                this.corrupt++;
+            return;
+        }
+        if (r.k === 'run' && typeof r.runId === 'string') {
+            this.runs.set(r.runId, { state: r.state === 'closed' ? 'closed' : 'open', at, strategy: r.strategy, accountId: typeof r.accountId === 'string' ? r.accountId : undefined, reason: typeof r.reason === 'string' ? r.reason : undefined });
+            return;
+        }
+        const e = this.byNonce.get(Number(r.nonce));
+        const name = r.ch;
+        const st = e?.ch.get(name);
+        if (!e || !st) {
+            if (!live && !['try', 'ack', 'rej', 'exp', 'drop'].includes(String(r.k)))
+                this.corrupt++;
+            return;
+        }
+        if (r.k === 'try') {
+            st.attempts = Number(r.attempt) || st.attempts + 1;
+            st.lastWhy = String(r.why ?? '').slice(0, 200);
+            st.nextAt = at + backoff(st.attempts);
+            return;
+        }
+        if (r.k === 'ack' || r.k === 'rej' || r.k === 'exp' || r.k === 'drop') {
+            st.done = r.k;
+            st.dup = r.dup === true;
+            st.lastWhy = typeof r.why === 'string' ? r.why.slice(0, 200) : st.lastWhy;
+        }
+    }
+    write(r) {
+        appendSync(this.path, JSON.stringify(r));
+        this.apply(r, true);
+    }
+    /** Move a fully settled outbox aside once it is large (the run ledgers keep every signal for the audit). */
+    compact() {
+        if (!existsSync(this.path) || statSync(this.path).size < COMPACT_ABOVE_BYTES)
+            return;
+        if (this.entries.some((e) => [...e.ch.values()].some((c) => !c.done)) || [...this.runs.values()].some((r) => r.state === 'open'))
+            return;
+        renameSync(this.path, join(realDir(this.home), `outbox.${this.now()}.done.jsonl`));
+        this.entries.length = 0;
+        this.byNonce.clear();
+        this.runs.clear();
+        // The fresh outbox starts with the nonce floor, so a lost nonce.json can never restart the count at 1.
+        if (this.lastNonce > 0)
+            this.write({ k: 'base', at: this.now(), lastNonce: this.lastNonce });
+    }
+    /** The next nonce, persisted before it is used (so a crash after this line can only skip a number, never reuse one). */
+    allocateNonce() {
+        const n = this.lastNonce + 1;
+        writeJsonAtomic(noncePath(this.home), { last: n, at: this.now() });
+        this.lastNonce = n;
+        return n;
+    }
+    get lastNonceIssued() { return this.lastNonce; }
+    /** Durably queue a signal for every channel set up now. Throws when the disk refuses (the caller then fires nothing). */
+    enqueue(signal) {
+        if (this.byNonce.has(signal.nonce))
+            throw new Error(`nonce ${signal.nonce} is already in the outbox`);
+        if (signal.nonce > this.lastNonce) {
+            writeJsonAtomic(noncePath(this.home), { last: signal.nonce, at: this.now() });
+            this.lastNonce = signal.nonce;
+        }
+        this.write({ k: 'enq', at: this.now(), channels: [...this.channels.keys()], signal });
+    }
+    markRun(runId, state, extra = {}) {
+        this.write({ k: 'run', at: this.now(), runId, state, ...extra });
+    }
+    /** Runs that were opened and never closed (their runner died before sending `halted`). */
+    unclosedRuns() {
+        return [...this.runs.entries()].filter(([, r]) => r.state === 'open').map(([runId, r]) => ({ runId, at: r.at, strategy: r.strategy, accountId: r.accountId }));
+    }
+    /**
+     * Deliver what is due. Serialized: a call while a pump runs queues ONE follow-up pump (signals queued meanwhile are
+     * covered) and resolves with its report.
+     */
+    pump() {
+        if (this.running) {
+            this.queued ??= this.running.then(() => { this.queued = null; return this.pump(); });
+            return this.queued;
+        }
+        this.running = this.pumpOnce().finally(() => { this.running = null; });
+        return this.running;
+    }
+    async pumpOnce() {
+        const rep = { acked: 0, retried: 0, rejected: 0, expired: 0, dropped: 0 };
+        const names = new Set();
+        for (const e of this.entries)
+            for (const [n, st] of e.ch)
+                if (!st.done)
+                    names.add(n);
+        // Channels deliver side by side (a slow hook never holds up the stream); each one strictly in nonce order.
+        await Promise.all([...names].map((name) => this.pumpChannel(name, rep)));
+        return rep;
+    }
+    async pumpChannel(name, rep) {
+        const chan = this.channels.get(name);
+        if (!chan && this.defer.has(name))
+            return;
+        let sent = 0;
+        for (const e of this.entries) {
+            const st = e.ch.get(name);
+            if (!st || st.done)
+                continue;
+            const now = this.now();
+            if (!chan) {
+                this.write({ k: 'drop', at: now, nonce: e.signal.nonce, ch: name, why: 'channel-not-configured' });
+                rep.dropped++;
+                continue;
+            }
+            if (now > e.signal.expiresAt) {
+                this.write({ k: 'exp', at: now, nonce: e.signal.nonce, ch: name });
+                rep.expired++;
+                continue;
+            }
+            if (st.nextAt > now)
+                return; // in order: a channel waits on its oldest undelivered signal
+            if (sent++ >= MAX_PER_PUMP)
+                return;
+            let r;
+            try {
+                r = await chan.deliver(e.signal, e.line);
+            }
+            catch (err) {
+                r = { ok: false, retry: true, why: err.message.slice(0, 160) };
+            }
+            const at = this.now();
+            const h = this.healthOf(name);
+            if (r.ok) {
+                this.write({ k: 'ack', at, nonce: e.signal.nonce, ch: name, ...(r.duplicate ? { dup: true } : {}) });
+                h.lastOkAt = at;
+                h.failing = 0;
+                rep.acked++;
+                continue;
+            }
+            h.lastErrAt = at;
+            h.lastErr = r.why.slice(0, 160);
+            h.failing++;
+            if (!r.retry) {
+                this.write({ k: 'rej', at, nonce: e.signal.nonce, ch: name, why: r.why.slice(0, 200) });
+                rep.rejected++;
+                continue;
+            }
+            this.write({ k: 'try', at, nonce: e.signal.nonce, ch: name, why: r.why.slice(0, 200), attempt: st.attempts + 1 });
+            rep.retried++;
+            return;
+        }
+    }
+    healthOf(name) {
+        let h = this.health.get(name);
+        if (!h) {
+            h = { name, pending: 0, lastOkAt: null, lastErrAt: null, lastErr: null, failing: 0 };
+            this.health.set(name, h);
+        }
+        return h;
+    }
+    /** Per channel: pending deliveries, the last success and failure (this process), consecutive failures. */
+    channelHealth() {
+        const out = [];
+        for (const name of new Set([...this.channels.keys(), ...this.health.keys()])) {
+            const h = { ...this.healthOf(name) };
+            h.pending = this.entries.filter((e) => { const s = e.ch.get(name); return s && !s.done; }).length;
+            if (h.lastErr === null) {
+                const last = [...this.entries].reverse().map((e) => e.ch.get(name)).find((s) => s?.lastWhy);
+                h.lastErr = last?.lastWhy ?? null;
+            }
+            out.push(h);
+        }
+        return out;
+    }
+    /** Signals with at least one channel not settled yet. */
+    depth() { return this.entries.filter((e) => [...e.ch.values()].some((s) => !s.done)).length; }
+    stateOf(nonce) {
+        const e = this.byNonce.get(nonce);
+        if (!e)
+            return null;
+        const ss = [...e.ch.values()];
+        if (!ss.length)
+            return this.now() > e.signal.expiresAt ? 'expired' : 'pending';
+        if (ss.some((s) => !s.done))
+            return ss.some((s) => s.done === 'ack') ? 'partial' : 'pending';
+        if (ss.length && ss.every((s) => s.done === 'ack'))
+            return 'delivered';
+        if (ss.some((s) => s.done === 'ack'))
+            return 'partial';
+        return ss.some((s) => s.done === 'rej') ? 'rejected' : 'expired';
+    }
+    /** Trade signals of a run that were queued or delivered and have not expired yet (the halted signal lists them). */
+    outstanding(runId) {
+        const now = this.now();
+        return this.entries.filter((e) => e.signal.type === 'trade' && (!runId || e.signal.runId === runId) && now <= e.signal.expiresAt).map((e) => e.signal.nonce);
+    }
+    signals() { return this.entries.map((e) => e.signal); }
+    get(nonce) { return this.byNonce.get(nonce)?.signal; }
+    findById(id) { return this.entries.find((e) => e.signal.id === id)?.signal; }
+}
+/**
+ * Every valid signal ever queued on this home, oldest nonce first: the live outbox plus the archives compaction moved
+ * aside. Read-only; used to rebuild the real holdings when held.json is missing.
+ */
+export function allOutboxSignals(home) {
+    const dir = realDir(home);
+    if (!existsSync(dir))
+        return [];
+    const byNonce = new Map();
+    for (const f of readdirSync(dir).filter((n) => /^outbox(\.\d+\.done)?\.jsonl$/.test(n))) {
+        for (const raw of readFileSync(join(dir, f), 'utf8').split('\n')) {
+            if (!raw.trim())
+                continue;
+            try {
+                const r = JSON.parse(raw);
+                if (r.k !== 'enq')
+                    continue;
+                const v = validateSignal(r.signal);
+                if (v.ok && !byNonce.has(v.signal.nonce))
+                    byNonce.set(v.signal.nonce, v.signal);
+            }
+            catch { /* torn line */ }
+        }
+    }
+    return [...byNonce.values()].sort((a, b) => a.nonce - b.nonce);
+}
+export const backoff = (attempt) => Math.min(BACKOFF_MAX_MS, BACKOFF_MIN_MS * 2 ** Math.max(0, attempt - 1));

package/dist/packages/autotrade/src/paper-limits.js

@@ -0,0 +1,87 @@
+// dexterm autotrade — the limits on a PAPER run (task-1654, 0.2.0 card E): the `paper` scope of limits.json.
+//
+// Paper ignores the limits unless the user adds a `modes.paper` scope; then a paper run (a forward test) obeys the
+// same guard a real run does (limits.ts checkBuy / checkSell / recordSell), so it shows what real mode would have
+// refused or clamped. The state is the run's own (in memory): paper never touches real mode's risk.json, held.json
+// or counters. No signal, no executor, no network: it only wraps the paper fill model.
+import { checkBuy, checkSell, freshRiskState, recordBuy, recordSell } from "./limits.js";
+import { localDay } from "./chat/audit.js";
+export class PaperLimitsExecutor {
+    name;
+    inner;
+    L;
+    solUsd;
+    bookUsd;
+    onTrip;
+    risk = null;
+    held = new Map();
+    constructor(o) {
+        this.inner = o.inner;
+        this.L = o.limits;
+        this.solUsd = o.solUsd;
+        this.bookUsd = o.bookUsd;
+        this.onTrip = o.onTrip ?? (() => { });
+        this.name = `paper-limits(${o.inner.name})`;
+    }
+    /** The run's own state, rolled at local midnight of the tick clock (a replay keeps its days). */
+    state(t) {
+        const day = localDay(t);
+        if (!this.risk)
+            this.risk = freshRiskState(day);
+        else if (this.risk.day !== day)
+            this.risk = { ...this.risk, day, buys: 0, buySol: 0, perCoin: {}, dayPnlSol: 0 };
+        this.risk.recentBuys = this.risk.recentBuys.filter((x) => t - x < 60_000);
+        this.risk.recentSells = this.risk.recentSells.filter((x) => t - x < 60_000);
+        return this.risk;
+    }
+    /** The maxHoldMinutes limit for the runner (RunOpts.maxHoldMinutes), or null. */
+    maxHoldMinutes() { return this.L.maxHoldMinutes ?? null; }
+    async buy(k, usd, ctx) {
+        const solUsd = this.solUsd();
+        if (!solUsd || !(solUsd > 0))
+            return { refused: 'paper-limits-sol-price-unknown' };
+        const s = this.state(k.t);
+        const prior = this.held.get(k.mint);
+        if (!prior && this.L.maxOpenPositions !== undefined && this.held.size >= this.L.maxOpenPositions)
+            return { refused: 'paper-limit-max-open' };
+        const openCostUsd = [...this.held.values()].reduce((a, h) => a + Math.max(0, h.costUsd - h.costOutUsd), 0);
+        const v = checkBuy(this.L, s, { now: k.t, solUsd, usd, bookUsd: this.bookUsd(), mint: k.mint, liqUsd: k.liq, mcapUsd: k.mcapResolved ?? k.mcap, positionCostUsd: prior ? Math.max(0, prior.costUsd - prior.costOutUsd) : 0, openCostUsd });
+        if (!v.ok)
+            return { refused: v.refused.replace(/^real-/, 'paper-limit-') };
+        const f = await this.inner.buy(k, v.usd, ctx);
+        if ('refused' in f)
+            return f;
+        recordBuy(s, k.mint, f.usd / solUsd, k.t);
+        const h = this.held.get(k.mint) ?? { tokens: 0, costUsd: 0, costOutUsd: 0, realizedUsd: 0 };
+        h.tokens += f.tokens;
+        h.costUsd += f.usd;
+        this.held.set(k.mint, h);
+        return v.clamps.length ? { ...f, limitClamps: v.clamps } : f;
+    }
+    async sell(k, tokens, reason) {
+        const s = this.state(k.t);
+        const cs = checkSell(this.L, s, k.t);
+        if (!cs.ok)
+            return { refused: cs.refused.replace(/^real-/, 'paper-limit-') };
+        const f = await this.inner.sell(k, tokens, reason);
+        if ('refused' in f)
+            return f;
+        const solUsd = this.solUsd() ?? 0;
+        const h = this.held.get(k.mint);
+        if (h && solUsd > 0) {
+            const frac = h.tokens > 0 ? Math.min(1, f.tokens / h.tokens) : 1;
+            const portion = Math.max(0, h.costUsd - h.costOutUsd) * frac;
+            h.tokens -= f.tokens;
+            h.costOutUsd += portion;
+            h.realizedUsd += Math.max(0, f.usd);
+            const closed = h.tokens <= 1e-12 || frac >= 0.999;
+            for (const line of recordSell(this.L, s, { pnlSol: (Math.max(0, f.usd) - portion) / solUsd, closedPnlSol: closed ? (h.realizedUsd - h.costUsd) / solUsd : null, now: k.t }))
+                this.onTrip(`PAPER LIMIT: ${line}`);
+            if (closed)
+                this.held.delete(k.mint);
+        }
+        else if (h && (h.tokens - f.tokens <= 1e-12))
+            this.held.delete(k.mint);
+        return f;
+    }
+}