@remnic/connector-x 9.69.64

package/dist/index.d.ts

@@ -0,0 +1,467 @@
+import { ConnectorApiError } from '@remnic/core/http-retry';
+
+/**
+ * Shared types for @remnic/connector-x.
+ */
+type XRecordKind = "bookmark" | "own_post";
+/** Trust gate for extracted memories. "suggest" routes through a review queue; "store" writes directly. */
+type XMemoryMode = "suggest" | "store";
+type XSourceKind = "mcp" | "corpusDir" | "cli";
+interface XAuthor {
+    id?: string;
+    username?: string;
+    name?: string;
+}
+interface XProvenance {
+    sourceId: string;
+    sourceKind: XSourceKind;
+    syncRunId: string;
+    fetchedAt: string;
+}
+/** Normalized record, the common currency every source emits. */
+interface XPostRecord {
+    postId: string;
+    kind: XRecordKind;
+    author?: XAuthor;
+    /** Post creation time (ISO 8601) when the source carries it. */
+    createdAt?: string;
+    /** When the bookmark act happened (ISO 8601), when known. */
+    bookmarkedAt?: string;
+    text: string;
+    urls: string[];
+    mediaCount: number;
+    /** Zero-credit enrichment (e.g. resolved URL titles from a local corpus). */
+    enrichment?: Record<string, unknown>;
+    provenance?: XProvenance;
+}
+/** The memory a record maps to, before trust gating. */
+interface XMemorySuggestion {
+    record: XPostRecord;
+    tags: string[];
+    category: string;
+    entityRef?: string;
+    confidence: number;
+    postUrl: string;
+    /** Composed memory sentence. */
+    content: string;
+}
+/**
+ * Host-provided memory sink. `submitSuggestion` feeds the review queue;
+ * `storeMemory` writes directly. The connector picks per `memoryMode`.
+ */
+interface XMemorySink {
+    submitSuggestion(suggestion: XMemorySuggestion): Promise<void>;
+    storeMemory(suggestion: XMemorySuggestion): Promise<void>;
+}
+interface XSourceFetchOutcome {
+    records: XPostRecord[];
+    /** Billable reads consumed (MCP only; 0 for zero-credit sources). */
+    reads: number;
+    pages: number;
+    /** Present when the source degraded instead of erroring. */
+    skipped?: {
+        reason: string;
+        detail?: string;
+    };
+}
+interface XSource {
+    id: string;
+    kind: XSourceKind;
+    fetch(ctx: XSourceFetchContext): Promise<XSourceFetchOutcome>;
+}
+interface XSourceFetchContext {
+    knownIds: ReadonlySet<string>;
+    budget: XBudgetRuntime;
+    signal?: AbortSignal;
+}
+/** Budget enforcement shared by the sync loop and the MCP source. */
+interface XBudgetRuntime {
+    canRead(): {
+        ok: true;
+    } | {
+        ok: false;
+        reason: string;
+        detail?: string;
+    };
+    noteRead(): void;
+    /** Pages consumed this sync by the current source. */
+    pagesUsed: number;
+    maxPages: number;
+}
+interface XSourceSyncSummary {
+    sourceId: string;
+    kind: XSourceKind;
+    recordsNew: number;
+    recordsKnown: number;
+    reads: number;
+    pages: number;
+    skipped?: {
+        reason: string;
+        detail?: string;
+    };
+    error?: string;
+}
+interface XSyncReport {
+    runId: string;
+    startedAt: string;
+    finishedAt: string;
+    memoryMode: XMemoryMode;
+    sources: XSourceSyncSummary[];
+    suggestionsSubmitted: number;
+    memoriesStored: number;
+    sinkFailures: number;
+    monthKey: string;
+    monthSpendUsd: number;
+}
+interface XSourceStatus {
+    sourceId: string;
+    kind: XSourceKind;
+    priority: number;
+    lastSyncAt: string | null;
+    lastRecordsNew: number;
+    available: boolean;
+    availabilityDetail?: string;
+}
+interface XStatusReport {
+    enabled: boolean;
+    memoryMode: XMemoryMode;
+    syncSchedule: string;
+    sources: XSourceStatus[];
+    seenCount: number;
+    monthKey: string;
+    monthSpendUsd: number;
+    monthlyCostCapUsd: number;
+    lastSyncAt: string | null;
+}
+
+/**
+ * Strict parser for the `xConnector` config block (issue #2009).
+ *
+ * Invalid values are rejected, never silently reinterpreted: unknown
+ * source kinds, duplicate source ids, priorities naming unknown sources,
+ * non-finite numbers, and unrecognized enum values all throw.
+ */
+
+declare const X_SOURCE_KINDS: readonly XSourceKind[];
+declare const X_MEMORY_MODES: readonly XMemoryMode[];
+declare const X_SYNC_SCHEDULES: readonly string[];
+declare const X_DEFAULT_MCP_URL = "https://api.x.com/mcp";
+declare const X_DEFAULT_TOKEN_FILE = "~/.openclaw/secrets/x-tokens.json";
+declare const X_DEFAULT_STATE_DIR = "~/.remnic/x-connector";
+/** Pay-per-use reference rate: ~1 credit per read at ~$0.01/credit. */
+declare const X_DEFAULT_COST_PER_READ_USD = 0.01;
+interface XBudgetConfig {
+    maxPagesPerSync: number;
+    maxCostUsdPerMonth: number;
+    costPerReadUsd: number;
+}
+interface XMcpSourceConfig {
+    id: string;
+    kind: "mcp";
+    url: string;
+    tokenFile: string;
+    bookmarksTool: string;
+    timelineTool: string;
+    maxResults: number;
+    budget: XBudgetConfig;
+}
+interface XCorpusSourceConfig {
+    id: string;
+    kind: "corpusDir";
+    path: string;
+}
+interface XCliSourceConfig {
+    id: string;
+    kind: "cli";
+    bin: string;
+    bookmarksArgs: string[];
+    /** When unset, this source contributes bookmarks only. */
+    postsArgs?: string[];
+}
+type XSourceConfig = XMcpSourceConfig | XCorpusSourceConfig | XCliSourceConfig;
+interface XConnectorConfig {
+    enabled: boolean;
+    userId?: string;
+    sources: XSourceConfig[];
+    sourcePriority: string[];
+    syncSchedule: string;
+    memoryMode: XMemoryMode;
+    stateDir: string;
+}
+declare class XConfigError extends Error {
+    constructor(message: string);
+}
+/** Coerces boolean-like strings at the config boundary; anything else is invalid. */
+declare function coerceXBool(value: unknown, field: string): boolean;
+declare function parseXConnectorConfig(raw: unknown): XConnectorConfig;
+/** OAuth2 client credentials for the MCP source, with env fallbacks. */
+declare function resolveMcpClientCredentials(source: XMcpSourceConfig, env?: NodeJS.ProcessEnv): {
+    clientId?: string;
+    clientSecret?: string;
+    tokenFile: string;
+};
+/** The effective monthly cost cap across all paid sources (max of per-source caps). */
+declare function monthlyCostCapUsd(config: XConnectorConfig): number;
+
+/**
+ * Canonical unknown-payload guard for @remnic/connector-x.
+ *
+ * Fields stay `unknown` after narrowing; every field read is checked at
+ * its use site with `typeof` / `in` / `Array.isArray`.
+ */
+declare function isXObject(value: unknown): value is Record<string, unknown>;
+
+/**
+ * Official X MCP client (Streamable HTTP, protocol 2025-06-18).
+ *
+ * Contract per https://docs.x.com (MCP, launched 2026-06-30): JSON-RPC
+ * over HTTP POST; `initialize` hands back an `Mcp-Session-Id` response
+ * header; response bodies may be plain JSON or SSE (`text/event-stream`,
+ * `data:` lines). Reads bill against X API credits — `credits depleted`
+ * (HTTP 402 or a 402-in-tool-result payload) maps to
+ * XCreditsDepletedError so callers can skip the cycle cleanly instead
+ * of erroring. Session `initialize`/`tools/list` are free.
+ *
+ * The API token is never logged and never included in thrown error
+ * messages.
+ */
+
+declare const X_MCP_PROTOCOL_VERSION = "2025-06-18";
+declare const X_MCP_DEFAULT_URL = "https://api.x.com/mcp";
+declare class XMcpError extends ConnectorApiError {
+    constructor(message: string, status?: number);
+}
+/** Clean-skip signal: the account's X API credits are exhausted. */
+declare class XCreditsDepletedError extends Error {
+    constructor();
+}
+interface XMcpToolCallResult {
+    isError: boolean;
+    /** text blocks of result.content, in order. */
+    texts: string[];
+    raw: unknown;
+}
+interface XMcpClientOptions {
+    url?: string;
+    /** Lazily supplies a valid bearer token (user-context OAuth2). */
+    tokenProvider: () => Promise<string>;
+    fetchImpl?: typeof fetch;
+    timeoutMs?: number;
+    sleep?: (ms: number) => Promise<void>;
+    protocolVersion?: string;
+    clientName?: string;
+    clientVersion?: string;
+}
+/** Parses an SSE body into decoded `data:` JSON values, in order. */
+declare function parseSseData(body: string): unknown[];
+/** True when a tool-result body signals exhausted credits (docs + observed shape). */
+declare function looksLikeCreditsDepleted(text: string): boolean;
+/** Extracts text blocks from an MCP tool-result content array. */
+declare function toolResultTexts(payload: Record<string, unknown>): string[];
+declare class XMcpClient {
+    private readonly url;
+    private readonly tokenProvider;
+    private readonly fetchImpl;
+    private readonly timeoutMs;
+    private readonly sleep;
+    private readonly protocolVersion;
+    private readonly clientName;
+    private readonly clientVersion;
+    private sessionId;
+    private nextMessageId;
+    constructor(options: XMcpClientOptions);
+    /**
+     * Calls an MCP tool. Re-initializes once when the server rejects the
+     * session id (e.g. expired session), then retries the call.
+     */
+    callTool(name: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<XMcpToolCallResult>;
+    /** Best-effort session shutdown (MCP `DELETE`). */
+    close(): Promise<void>;
+    private withSessionRetry;
+    private ensureSession;
+    private allocateId;
+    private rpcMessage;
+    private decodeBody;
+}
+
+/**
+ * Payload normalization for all three X sources plus the shared
+ * dedupe fingerprint and the record → memory mapping.
+ *
+ * All parsers are shape-tolerant: X MCP payloads follow the v2 API
+ * expansions shape (`data[]` + `includes.users`), local corpora and
+ * CLI tools use a variety of field aliases. Every field read is
+ * checked; unrecognized shapes yield empty results, never crashes.
+ */
+
+/** Sorts keys recursively so key order never changes the fingerprint. */
+declare function stableStringify(value: unknown): string;
+/** Identity fingerprint: same post + same content = same memory, regardless of source key order. */
+declare function recordFingerprint(record: XPostRecord): string;
+/**
+ * Normalizes an MCP tool-result payload (v2 expansions shape) into
+ * records. Accepts `{data: [...]}` with `includes.users`, a bare
+ * array, or `{bookmarks: [...]}`.
+ */
+declare function normalizeMcpPayload(payload: unknown, kind: XRecordKind, ownUsername?: string): XPostRecord[];
+/** Normalizes one corpus/CLI entry (tolerant field aliases). */
+declare function normalizeCorpusEntry(entry: unknown, fallbackKind: XRecordKind, ownUsername?: string): XPostRecord | null;
+/**
+ * Record → memory mapping (issue #2009 §2):
+ * - bookmarks → tag `x/bookmark`, category `reference` (carries a URL) or `interest`
+ * - own posts → tag `x/post`, category `expression`, higher confidence
+ */
+declare function suggestionForRecord(record: XPostRecord): XMemorySuggestion;
+
+/**
+ * Pluggable X sources (issue #2009): official X MCP (paid, budget-
+ * capped), a local corpus directory (zero credits), and a cookie-CLI
+ * such as `bird` (zero credits). All three emit the same normalized
+ * XPostRecord currency and degrade with a `skipped` reason instead of
+ * throwing, except auth/config breakage which surfaces to the sync
+ * report's `error` field.
+ */
+
+type XExecFn = (bin: string, args: string[]) => Promise<{
+    stdout: string;
+    stderr: string;
+}>;
+interface XSourceDeps {
+    /** Owning X user id — gates the own-posts timeline reads for MCP sources. */
+    userId?: string;
+    fetchImpl?: typeof fetch;
+    sleep?: (ms: number) => Promise<void>;
+    now?: () => number;
+    env?: NodeJS.ProcessEnv;
+    execImpl?: XExecFn;
+}
+/** Budget enforcement for paid (MCP) sources. */
+declare class XBudgetTracker implements XBudgetRuntime {
+    private readonly budget;
+    private readonly monthSpendUsd;
+    pagesUsed: number;
+    reads: number;
+    readonly maxPages: number;
+    constructor(budget: XBudgetConfig, monthSpendUsd: number);
+    canRead(): {
+        ok: true;
+    } | {
+        ok: false;
+        reason: string;
+        detail?: string;
+    };
+    noteRead(): void;
+}
+/** Zero-credit sources never consume budget. */
+declare const unlimitedBudget: XBudgetRuntime;
+/** Builds the source adapter for a parsed source config entry. */
+declare function createXSource(config: XMcpSourceConfig | XCorpusSourceConfig | XCliSourceConfig, deps?: XSourceDeps): XSource;
+
+/**
+ * Sync orchestration (issue #2009): run sources in configured priority
+ * order (cheapest first is the recommended arrangement), dedupe by
+ * post_id + content fingerprint, stamp provenance, and route the mapped
+ * memory through the trust gate (`suggest` → review queue, `store` →
+ * direct write). Persisted state lives in `<stateDir>/state.json`
+ * (dedupe map, per-source last sync + new counts, monthly cost ledger);
+ * every ingested record is materialized under `<stateDir>/records/`.
+ */
+
+interface XSyncDeps extends XSourceDeps {
+    sink: XMemorySink;
+}
+/** Runs one sync cycle. Source-level degradation lands in the report, never a throw. */
+declare function runXSync(config: XConnectorConfig, deps: XSyncDeps): Promise<XSyncReport>;
+/**
+ * Offline status snapshot: config sources, availability, spend vs cap.
+ * No network calls, no credit use.
+ */
+declare function getXStatus(config: XConnectorConfig, deps?: Pick<XSyncDeps, "execImpl" | "env">): Promise<XStatusReport>;
+
+/**
+ * OAuth2 user-token store with single-owner refresh for the X MCP
+ * source.
+ *
+ * X rotates the refresh token on EVERY refresh. Two independent
+ * refreshers fork the chain and kill one of them (observed failure:
+ * HTTP 401 on refresh after two tools both rotated the same grant).
+ * This store is therefore the single owner of the refresh chain: the
+ * refresh runs only while holding `${tokenFile}.lock`; a concurrent
+ * refresher waits, then adopts the rotated pair from the file — it
+ * never refreshes in parallel.
+ *
+ * Token files are written 0600, atomic (tmp + rename), and unknown
+ * top-level fields are preserved on rotation.
+ */
+declare const X_TOKEN_REFRESH_URL = "https://api.x.com/2/oauth2/token";
+declare class XTokenError extends Error {
+    constructor(message: string);
+}
+/** The refresh chain forked or was revoked — re-authorization is required. */
+declare class XRefreshChainBrokenError extends XTokenError {
+    constructor(detail: string);
+}
+interface XTokenPair {
+    accessToken: string;
+    refreshToken: string;
+    /** Epoch ms when the access token expires. */
+    expiresAt: number;
+}
+interface XTokenStoreOptions {
+    tokenFile: string;
+    clientId: string;
+    clientSecret: string;
+    /** OAuth2 token endpoint. */
+    refreshUrl?: string;
+    fetchImpl?: typeof fetch;
+    now?: () => number;
+    sleep?: (ms: number) => Promise<void>;
+    lockStaleMs?: number;
+    lockWaitMs?: number;
+}
+declare class XTokenStore {
+    private readonly tokenFile;
+    private readonly clientId;
+    private readonly clientSecret;
+    private readonly refreshUrl;
+    private readonly fetchImpl;
+    private readonly now;
+    private readonly sleep;
+    private readonly lockStaleMs;
+    private readonly lockWaitMs;
+    private cached;
+    constructor(options: XTokenStoreOptions);
+    /** Returns a valid access token, refreshing under the lock when expired. */
+    getAccessToken(): Promise<string>;
+    private getValidPair;
+    /**
+     * Refreshes the token pair under the file lock. When another owner
+     * holds the lock, waits for it, then adopts the pair it wrote.
+     */
+    refresh(): Promise<XTokenPair>;
+    private refreshWithLock;
+    private waitForOtherOwner;
+    private acquireLock;
+    /** Steals the lock when its mtime is older than lockStaleMs. */
+    private stealStaleLock;
+    private requestRefresh;
+    private readTokenFile;
+    /** Atomic 0600 write; preserves unknown top-level fields from the prior file. */
+    private writeTokenFile;
+}
+
+/**
+ * Default on-disk memory sink: `suggest` mode writes review-queue
+ * files under `<stateDir>/suggestions/`, `store` mode writes directly
+ * under `<stateDir>/records/`. Hosts with a live Remnic daemon pass
+ * their own XMemorySink instead.
+ */
+
+interface FileSinkOptions {
+    stateDir: string;
+    mode: "suggest" | "store";
+}
+/** On-disk sink honoring the memoryMode trust gate by directory. */
+declare function createFileSink(options: FileSinkOptions): XMemorySink;
+
+export { type XAuthor, type XBudgetConfig, type XBudgetRuntime, XBudgetTracker, type XCliSourceConfig, XConfigError, type XConnectorConfig, type XCorpusSourceConfig, XCreditsDepletedError, type XExecFn, XMcpClient, type XMcpClientOptions, XMcpError, type XMcpSourceConfig, type XMcpToolCallResult, type XMemoryMode, type XMemorySink, type XMemorySuggestion, type XPostRecord, type XProvenance, type XRecordKind, XRefreshChainBrokenError, type XSource, type XSourceConfig, type XSourceDeps, type XSourceFetchContext, type XSourceFetchOutcome, type XSourceKind, type XSourceStatus, type XSourceSyncSummary, type XStatusReport, type XSyncDeps, type XSyncReport, XTokenError, type XTokenPair, XTokenStore, type XTokenStoreOptions, X_DEFAULT_COST_PER_READ_USD, X_DEFAULT_MCP_URL, X_DEFAULT_STATE_DIR, X_DEFAULT_TOKEN_FILE, X_MCP_DEFAULT_URL, X_MCP_PROTOCOL_VERSION, X_MEMORY_MODES, X_SOURCE_KINDS, X_SYNC_SCHEDULES, X_TOKEN_REFRESH_URL, coerceXBool, createFileSink, createXSource, getXStatus, isXObject, looksLikeCreditsDepleted, monthlyCostCapUsd, normalizeCorpusEntry, normalizeMcpPayload, parseSseData, parseXConnectorConfig, recordFingerprint, resolveMcpClientCredentials, runXSync, stableStringify, suggestionForRecord, toolResultTexts, unlimitedBudget };

package/dist/index.js

@@ -0,0 +1,78 @@
+// openclaw-engram: Local-first memory plugin
+import {
+  XBudgetTracker,
+  XConfigError,
+  XCreditsDepletedError,
+  XMcpClient,
+  XMcpError,
+  XRefreshChainBrokenError,
+  XTokenError,
+  XTokenStore,
+  X_DEFAULT_COST_PER_READ_USD,
+  X_DEFAULT_MCP_URL,
+  X_DEFAULT_STATE_DIR,
+  X_DEFAULT_TOKEN_FILE,
+  X_MCP_DEFAULT_URL,
+  X_MCP_PROTOCOL_VERSION,
+  X_MEMORY_MODES,
+  X_SOURCE_KINDS,
+  X_SYNC_SCHEDULES,
+  X_TOKEN_REFRESH_URL,
+  coerceXBool,
+  createFileSink,
+  createXSource,
+  getXStatus,
+  isXObject,
+  looksLikeCreditsDepleted,
+  monthlyCostCapUsd,
+  normalizeCorpusEntry,
+  normalizeMcpPayload,
+  parseSseData,
+  parseXConnectorConfig,
+  recordFingerprint,
+  resolveMcpClientCredentials,
+  runXSync,
+  stableStringify,
+  suggestionForRecord,
+  toolResultTexts,
+  unlimitedBudget
+} from "./chunk-JR2ZNAYD.js";
+export {
+  XBudgetTracker,
+  XConfigError,
+  XCreditsDepletedError,
+  XMcpClient,
+  XMcpError,
+  XRefreshChainBrokenError,
+  XTokenError,
+  XTokenStore,
+  X_DEFAULT_COST_PER_READ_USD,
+  X_DEFAULT_MCP_URL,
+  X_DEFAULT_STATE_DIR,
+  X_DEFAULT_TOKEN_FILE,
+  X_MCP_DEFAULT_URL,
+  X_MCP_PROTOCOL_VERSION,
+  X_MEMORY_MODES,
+  X_SOURCE_KINDS,
+  X_SYNC_SCHEDULES,
+  X_TOKEN_REFRESH_URL,
+  coerceXBool,
+  createFileSink,
+  createXSource,
+  getXStatus,
+  isXObject,
+  looksLikeCreditsDepleted,
+  monthlyCostCapUsd,
+  normalizeCorpusEntry,
+  normalizeMcpPayload,
+  parseSseData,
+  parseXConnectorConfig,
+  recordFingerprint,
+  resolveMcpClientCredentials,
+  runXSync,
+  stableStringify,
+  suggestionForRecord,
+  toolResultTexts,
+  unlimitedBudget
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@remnic/connector-x",
+  "version": "9.69.64",
+  "description": "X (Twitter) connector for Remnic — remember the user's posts and bookmarks via the official X MCP, a local corpus, or a CLI",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "bin": {
+    "remnic-x": "./dist/cli.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "provenance": true
+  },
+  "peerDependencies": {
+    "@remnic/core": "^9.69.64"
+  },
+  "devDependencies": {
+    "tsup": "^8.0.0",
+    "typescript": "^5.7.0",
+    "tsx": "^4.0.0",
+    "@remnic/core": "9.69.64"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/joshuaswarren/remnic.git",
+    "directory": "packages/connector-x"
+  },
+  "keywords": [
+    "remnic",
+    "memory",
+    "x",
+    "twitter",
+    "bookmarks",
+    "mcp",
+    "connector"
+  ],
+  "scripts": {
+    "build": "tsup src/index.ts src/cli.ts --format esm --dts",
+    "precheck-types": "node ../../scripts/ensure-bench-build-deps.mjs",
+    "check-types": "tsc --noEmit",
+    "test": "NODE_OPTIONS=\"${NODE_OPTIONS:+$NODE_OPTIONS }--conditions=remnic-source\" tsx --test src/config.test.ts src/mcp-client.test.ts src/token-store.test.ts src/normalize.test.ts src/sources.test.ts src/sync.test.ts"
+  }
+}