@tinycloud/courier 0.2.0-beta.41 → 0.2.0-beta.42

package/README.md

@@ -676,6 +676,71 @@ The service runs the same command line you gave (install flags removed, `claude`
 profile on the same machine. systemd needs `loginctl enable-linger $USER` to
 keep the watcher running while logged out.
 
+## Intake reader
+
+`courier intake` reads the mirror of paired peers' messages from the self-space KV
+and writes one JSON file per message into a queue directory for the intake reader.
+It does not send anything, and it does not change the watcher.
+
+```bash
+courier intake --once --queue-dir <dir> --state-dir <dir> --peers-file <file>
+courier intake --interval 60 --queue-dir <dir> --state-dir <dir> --peers-file <file>
+```
+
+Defaults for `--queue-dir`, `--state-dir` and `--peers-file` are under
+`<COURIER_HOME>/intake/`. Only peers with status `paired` are read. `intake.lock` in the state directory allows one
+intake process per state directory: a live holder always refuses a second start, a lock whose pid is gone is broken,
+and an unreadable lock refuses.
+
+Queue record (`<queue-dir>/<peerId>/<msgId>.json`): peer queue directories are created
+with mode 2750 (setgid) and only directories created by that call are chmodded. Records
+are mode 0640, written atomically without overwriting.
+
+`{v:1, msgId, peerId, threadId, sender:{id, displayName}, receivedAt, text, cheapChecks:{textVerdict, tier, peerRatePerHourRemaining, flags}}`
+
+`peerId` is the peer handle and equals the queue directory name.
+
+`receivedAt` is the local time the record was written. `text` is the message text,
+cut to 16 KiB UTF-8 at a code-point boundary when longer; the `size` flag marks a cut.
+
+Flags, in this order: `too-long`, `link`, `instruction`, `request`, `secret-request`,
+`encoded`, `unicode`, `size`, `rate`. They are cheap heuristics for the reader, not
+a verdict on content.
+
+Ordering: each cycle reads at most 200 mirror messages per peer. A cyclic cursor in `intake-state.json` marks the last
+id picked; the next cycle takes the next 200 ids after it in ascending order and wraps to the oldest id after the
+newest. Ids more than 7 days old by message time are dropped before selection: they are not fetched and get no marker,
+and a queue record that old moves to `done/`. A message is skipped when a queue, done or overflow file for its id
+already exists. A message id more than 10 minutes ahead of local time is not recorded and does not block the others:
+the cursor moves past it, so it is retried only when the cursor next wraps around the mirror. A mirror key whose
+message or thread id is not a ULID is logged as `intake-skip` (`bad-id`) and gets no marker. A message id whose
+time cannot be decoded gets a `bad-id` done marker. A fetched row that is malformed, or whose `messageId` or
+`contextId` does not match its key, gets a `mismatch` done marker. A message whose processing throws is logged
+as `intake-error`, and the cursor still moves past it. Existing history is not queued if you run
+`courier intake --init-from-now` first. It marks the current mirror messages of each paired peer as done (outcome
+`init`), and it refuses to run again while a paired peer has done entries unless `--force` is given. Without it, the
+first cycle works through the backlog from the oldest id inside the 7-day window, 200 rows per cycle.
+
+Rate: each peer is limited to 100 inbound messages per hour by default, including
+peers absent from `peers.json`. Over the cap, the message is not queued. It is written
+to the overflow directory (`<COURIER_HOME>/intake/overflow/<peerId>/`, directories 0700,
+files 0600), where it is kept until its message is 7 days old by message time. Each peer's overflow is capped at 200
+files and 1 MiB; the oldest messages are dropped first. Each cycle retires at most 500 expired files per directory.
+Over-cap retirement runs once per peer per pass: it removes at most 500 files by count, then, once at most 200 files
+remain, removes the oldest of them until the 1 MiB cap holds, so one pass can retire up to 700 files. A cycle runs the
+pass from the janitor and again after it writes new overflow for that peer, so one cycle can retire more than 500
+over-cap files for one peer. Each retirement writes an `overflow` done marker before the unlink, so the id is not admitted
+again when the cursor laps. The first overflow in each hour also writes one `rate-<unixHour>` record to the queue with
+`peerRatePerHourRemaining: 0` and the `rate` flag. After a start, the hour's count is rebuilt from `receivedAt` in
+queue and done records whose files were modified within the last hour.
+
+`peers.json` must not be group- or world-writable and maps a peer id to
+`{"tier": "trusted"|"strict", "ratePerHour": n, "outPerHour": n}`. `tier` defaults to
+`strict` for absent peers. `ratePerHour` and `outPerHour` are integers from 1 up to the
+default of 100. A value outside that range, or any other invalid entry, makes the whole
+file invalid: startup refuses to run, and a running cycle keeps its last good map and
+logs `intake-error`.
+
 ## Authority
 
 Courier accepts only an OpenKey grant whose signed authority exactly equals

package/dist/intake/flags.d.ts

@@ -0,0 +1,48 @@
+import type { Tier } from './peers.js';
+/**
+ * The complete set of flags the reader may attach to a queue record. Nothing else is emitted; the `Flag` type is derived
+ * from these keys, so an unlisted flag does not compile. `severity` is `hard`, `soft`, or `tiered`. A `tiered` flag is soft
+ * for the trusted tier and hard for every other tier; `severityFor` resolves it. The reader records the flags; the gate
+ * decides what each severity means.
+ */
+export declare const FLAG_TABLE: {
+    readonly 'too-long': {
+        readonly severity: "tiered";
+        readonly description: "text is over 8000 UTF-16 units; the text is still queued";
+    };
+    readonly link: {
+        readonly severity: "soft";
+        readonly description: "a URL-like string is present";
+    };
+    readonly instruction: {
+        readonly severity: "soft";
+        readonly description: "matches the fixed instruction-phrase table";
+    };
+    readonly request: {
+        readonly severity: "soft";
+        readonly description: "asks the reader or its principal to take an action (fixed verb-phrase table)";
+    };
+    readonly 'secret-request': {
+        readonly severity: "hard";
+        readonly description: "asks for keys, tokens, passwords or credentials";
+    };
+    readonly encoded: {
+        readonly severity: "hard";
+        readonly description: "a long base64- or hex-looking run is present";
+    };
+    readonly unicode: {
+        readonly severity: "hard";
+        readonly description: "shared text verdict: not-nfc, disallowed-code-point, edge-whitespace, not-string or empty";
+    };
+    readonly size: {
+        readonly severity: "hard";
+        readonly description: "original text exceeded the 16 KiB queue cap and was cut at a code-point boundary";
+    };
+    readonly rate: {
+        readonly severity: "hard";
+        readonly description: "peer is over its inbound per-hour cap";
+    };
+};
+export type Flag = keyof typeof FLAG_TABLE;
+/** The severity a flag has for a peer's tier. `tiered` flags are soft for `trusted` and hard otherwise. */
+export declare function severityFor(flag: Flag, tier: Tier): 'hard' | 'soft';

package/dist/intake/flags.js

@@ -0,0 +1,25 @@
+/**
+ * The complete set of flags the reader may attach to a queue record. Nothing else is emitted; the `Flag` type is derived
+ * from these keys, so an unlisted flag does not compile. `severity` is `hard`, `soft`, or `tiered`. A `tiered` flag is soft
+ * for the trusted tier and hard for every other tier; `severityFor` resolves it. The reader records the flags; the gate
+ * decides what each severity means.
+ */
+export const FLAG_TABLE = {
+    'too-long': { severity: 'tiered', description: 'text is over 8000 UTF-16 units; the text is still queued' },
+    link: { severity: 'soft', description: 'a URL-like string is present' },
+    instruction: { severity: 'soft', description: 'matches the fixed instruction-phrase table' },
+    request: { severity: 'soft', description: 'asks the reader or its principal to take an action (fixed verb-phrase table)' },
+    'secret-request': { severity: 'hard', description: 'asks for keys, tokens, passwords or credentials' },
+    encoded: { severity: 'hard', description: 'a long base64- or hex-looking run is present' },
+    unicode: { severity: 'hard', description: 'shared text verdict: not-nfc, disallowed-code-point, edge-whitespace, not-string or empty' },
+    size: { severity: 'hard', description: 'original text exceeded the 16 KiB queue cap and was cut at a code-point boundary' },
+    rate: { severity: 'hard', description: 'peer is over its inbound per-hour cap' },
+};
+/** The severity a flag has for a peer's tier. `tiered` flags are soft for `trusted` and hard otherwise. */
+export function severityFor(flag, tier) {
+    const severity = FLAG_TABLE[flag].severity;
+    if (severity !== 'tiered')
+        return severity;
+    return tier === 'trusted' ? 'soft' : 'hard';
+}
+//# sourceMappingURL=flags.js.map

package/dist/intake/flags.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flags.js","sourceRoot":"","sources":["../../src/intake/flags.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,UAAU,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,0DAA0D,EAAE;IAC3G,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,8BAA8B,EAAE;IACvE,WAAW,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,4CAA4C,EAAE;IAC5F,OAAO,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,8EAA8E,EAAE;IAC1H,gBAAgB,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,iDAAiD,EAAE;IACtG,OAAO,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,8CAA8C,EAAE;IAC1F,OAAO,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,2FAA2F,EAAE;IACvI,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,kFAAkF,EAAE;IAC3H,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,uCAAuC,EAAE;CACgB,CAAC;AAInG,2GAA2G;AAC3G,MAAM,UAAU,WAAW,CAAC,IAAU,EAAE,IAAU;IAChD,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC;IAC3C,IAAI,QAAQ,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC3C,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;AAC9C,CAAC"}

package/dist/intake/intake.d.ts

@@ -0,0 +1,71 @@
+import type { ParsedArgs } from '../args.js';
+import type { KV } from '../inbox.js';
+import { type Profile } from '../profile.js';
+import { type TextVerdict } from '../text.js';
+import { type WatchLog } from '../wake.js';
+import { type Flag } from './flags.js';
+import { type Peers } from './peers.js';
+import { type IntakeState } from './queue.js';
+/** Queued text is capped at this many UTF-16 units; longer text is flagged `too-long`. */
+export declare const MAX_QUEUE_CHARS = 8000;
+export interface IntakeRecord {
+    v: 1;
+    msgId: string;
+    peerId: string;
+    threadId: string;
+    sender: {
+        id: string;
+        displayName: string;
+    };
+    receivedAt: string;
+    text: string;
+    cheapChecks: {
+        textVerdict: TextVerdict;
+        tier: string;
+        peerRatePerHourRemaining: number;
+        flags: Flag[];
+    };
+}
+export interface CycleInput {
+    kv: KV;
+    profile: Profile;
+    peers: Peers;
+    queueRoot: string;
+    doneRoot: string;
+    overflowRoot: string;
+    stateDir: string;
+    state: IntakeState;
+    now: () => number;
+    log: WatchLog;
+    /** Mirror ids taken per peer per cycle. Defaults to MIRROR_BATCH. */
+    batch?: number;
+}
+/** Cuts `text` to at most `maxBytes` of UTF-8 at a code-point boundary. */
+export declare function capUtf8(text: string, maxBytes: number): {
+    text: string;
+    truncated: boolean;
+};
+/** Maps the verdict and the cheap text detectors to flag codes from FLAG_TABLE, in table order. */
+export declare function flagsFor(input: {
+    text: string;
+    verdict: TextVerdict;
+    overRate: boolean;
+    truncated: boolean;
+}): Flag[];
+/** One pass: janitor, then every paired peer's mirror. Returns how many new queue files were written (rate records included). */
+export declare function intakeCycle(input: CycleInput): Promise<number>;
+/** Input for `--init-from-now`: writes done markers only, never queue records. */
+export interface InitInput {
+    kv: KV;
+    profile: Profile;
+    doneRoot: string;
+    now: () => number;
+    log: WatchLog;
+}
+/**
+ * Marks every mirrored message of each paired peer as done (outcome `init`), so the first real intake skips existing
+ * history. Refuses before writing anything when a paired peer already has a done/ entry, unless `force`. Existing markers
+ * are not rewritten. Returns the number of markers written per peer.
+ */
+export declare function initFromNow(input: InitInput, force: boolean): Promise<Map<string, number>>;
+export declare function cmdIntake(args: ParsedArgs): Promise<void>;