dexterm-cli 0.0.0-stage → 0.1.2

package/dist/packages/data/src/social/chain.js

@@ -0,0 +1,83 @@
+// @dexterm/data — on-chain "social" rows (task-1330; design §4 table row "on-chain social", §5.2).
+// The truth layer that confirms or contradicts attention, rendered as posts with `platform: 'chain'`.
+// Rows are RULE-DERIVED from data the app already has (labelled top holders, the trade tape); the text is
+// ours (templates + numbers), the only untrusted parts are label words and short addresses, and even those
+// go through the ingest firewall like every other post. task-1359: the wallet-intel scan (task-1340) feeds its
+// linked clusters (R1 funder / R2 bundle / R3 transfer / R4 dev-linked) and the launch-window snipers through the
+// same ChainSignal shape (chainSignalsFromIntel).
+export const LARGE_TRADE_USD = 5_000;
+export const CHAIN_ROWS_MAX = 12;
+const short = (a) => (a.length > 10 ? `${a.slice(0, 4)}…${a.slice(-4)}` : a);
+const usd = (n) => (n >= 1e6 ? `$${(n / 1e6).toFixed(2)}M` : n >= 1e3 ? `$${(n / 1e3).toFixed(1)}K` : `$${n.toFixed(0)}`);
+/** Template the row text from OUR words; numbers formatted here, never free text from a source. */
+export function chainSignalText(s) {
+    const who = s.label ? `${s.label} wallet ${short(s.wallet)}` : `wallet ${short(s.wallet)}`;
+    switch (s.kind) {
+        case 'kol-buy': return `${who} bought${s.usd !== null ? ` ${usd(s.usd)}` : ''} (tracked wallet, on-chain)`;
+        case 'labelled-holder': return `${who} holds ${s.pct !== null ? `${s.pct.toFixed(2)}%` : 'a top-20 slot'} of supply`;
+        case 'large-trade': return `${who} ${s.side === 'sell' ? 'sold' : 'bought'} ${s.usd !== null ? usd(s.usd) : ''} in one trade`.replace(/\s+/g, ' ').trim();
+        case 'dev-move': return `${who} moved ${s.pct !== null ? `${s.pct.toFixed(1)}%` : 'tokens'}${s.side === 'sell' ? ' out (sell)' : ''}`;
+        case 'cluster': {
+            // label = the cluster kind the wallet-intel rules assigned (rules.ts R1–R4); text says which rule linked them
+            const share = s.pct !== null ? `${s.pct.toFixed(1)}% of supply` : 'a share of supply';
+            const n = s.count !== undefined ? `${s.count} wallets` : 'linked wallets';
+            const why = s.label === 'bundle' ? 'bought in the same slot and share a funder (bundle, R2)'
+                : s.label === 'dev-linked' ? 'were funded by the creator wallet (dev-linked, R4)'
+                    : s.label === 'transfer' ? 'are linked by direct token transfers at launch (R3)'
+                        : 'share one funding wallet (R1)';
+            return `${share} sits in ${n} that ${why}`;
+        }
+        case 'snipers': return `${s.count ?? 0} sniper wallet${s.count === 1 ? '' : 's'} took ${s.pct !== null ? `${s.pct.toFixed(1)}%` : 'part'} of the launch window (first 3 slots)`;
+    }
+}
+export function chainSignalToRaw(s, fetchedAt) {
+    const tier = s.label === 'kol' || s.label === 'smart' ? 'kol-b' : 'none';
+    return {
+        id: `chain:${s.mint.slice(0, 8)}:${s.kind}:${s.wallet.slice(0, 8)}:${Math.floor(s.t / 1000)}`,
+        t: s.t,
+        platform: 'chain',
+        author: { id: s.wallet, handle: short(s.wallet), tier, createdAt: null },
+        text: chainSignalText(s),
+        url: null,
+        lang: null,
+        metrics: {},
+        mints: [s.mint],
+        source: 'solana-rpc',
+        fetchedAt,
+    };
+}
+/**
+ * Rule-derived signals from the open token's panels: labelled top holders (dev / sniper / smart; LP
+ * and CEX skipped — they are plumbing, not people), tracked-wallet buys, and large single trades.
+ */
+export function chainSignalsFromToken(input) {
+    const out = [];
+    const large = input.largeTradeUsd ?? LARGE_TRADE_USD;
+    for (const h of input.holders) {
+        if (!h.label || h.label === 'LP' || h.label === 'cex' || h.label === 'bridge')
+            continue;
+        out.push({ mint: input.mint, kind: 'labelled-holder', wallet: h.address, label: h.label, side: null, usd: null, pct: h.pct, t: input.now, source: 'holders' });
+    }
+    for (const tr of input.trades) {
+        const kol = input.kolWallets?.has(tr.wallet.toLowerCase()) ?? false;
+        if (kol && tr.side === 'buy')
+            out.push({ mint: input.mint, kind: 'kol-buy', wallet: tr.wallet, label: 'kol', side: 'buy', usd: tr.usd, pct: null, t: tr.t, source: 'tape' });
+        else if (tr.usd >= large)
+            out.push({ mint: input.mint, kind: 'large-trade', wallet: tr.wallet, label: null, side: tr.side, usd: tr.usd, pct: null, t: tr.t, source: 'tape' });
+    }
+    out.sort((a, b) => b.t - a.t || (b.usd ?? 0) - (a.usd ?? 0));
+    return out.slice(0, input.max ?? CHAIN_ROWS_MAX);
+}
+/** Rule-derived rows from a wallet-intel scan: linked clusters (largest first) and the launch-window snipers. */
+export function chainSignalsFromIntel(input) {
+    const out = [];
+    for (const c of input.clusters) {
+        if (c.kind === 'single' || c.members.length < 2 || !c.members[0])
+            continue;
+        out.push({ mint: input.mint, kind: 'cluster', wallet: c.members[0], label: c.kind, side: null, usd: null, pct: c.pct, t: input.t, source: 'wallet-intel', count: c.members.length });
+    }
+    out.sort((a, b) => (b.pct ?? 0) - (a.pct ?? 0));
+    if (input.snipers && input.snipers.count > 0)
+        out.unshift({ mint: input.mint, kind: 'snipers', wallet: 'snipers', label: null, side: null, usd: null, pct: input.snipers.launchWindowSharePct, t: input.t, source: 'wallet-intel', count: input.snipers.count });
+    return out.slice(0, input.max ?? 6);
+}

package/dist/packages/data/src/social/dedupe.js

@@ -0,0 +1,205 @@
+// @dexterm/data — dedupe and spam/bot rules (task-1330; design §5.5). All local, all deterministic,
+// testable on recorded fixtures. Nothing here looks at follower counts.
+//
+//  1. Exact + near-duplicate text (normalized, URL/mint stripped, SimHash ≤ 3 bits) within 10 min → one
+//     cluster row with `dupCount`; the cluster is a single vote for velocity (members are `hidden`).
+//  2. Account burst: ≥ 10 posts about one mint from accounts created within 30 days, inside 2 min → the
+//     posts are `spam`, excluded from velocity, shown collapsed on the head.
+//  3. Cashtag squats: a `$SYM` whose text carries a different base58 address than the row's mint is
+//     `squat`; a symbol with ≥ 3 live mints and no resolved mint is `unverified-cashtag`.
+export const DEDUPE_WINDOW_MS = 10 * 60_000;
+export const DEDUPE_MAX_BITS = 3;
+export const BURST_MIN_POSTS = 10;
+export const BURST_SPAN_MS = 2 * 60_000;
+export const FRESH_ACCOUNT_MS = 30 * 86_400_000;
+export const SQUAT_MIN_LIVE_MINTS = 3;
+/** Lower-case, drop `[url]`/`[addr]` placeholders, cashtags, @handles, punctuation and digits-only tokens; collapse spaces. */
+export function normalizeForDedupe(text) {
+    return text
+        .toLowerCase()
+        .replace(/\b(?:https?:\/\/|www\.)\S+/g, ' ')
+        .replace(/\[(?:url|addr)\]/g, ' ')
+        .replace(/\$[a-z][a-z0-9_]*/g, ' ')
+        .replace(/@[a-z0-9_]+/g, ' ')
+        .replace(/[^\p{L}\p{N}\s]/gu, ' ')
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+const FNV_OFFSET = 0xcbf29ce484222325n;
+const FNV_PRIME = 0x100000001b3n;
+const MASK64 = (1n << 64n) - 1n;
+/** FNV-1a 64 over UTF-16 code units (deterministic across platforms, no deps). */
+export function fnv1a64(s) {
+    let h = FNV_OFFSET;
+    for (let i = 0; i < s.length; i++) {
+        h ^= BigInt(s.charCodeAt(i));
+        h = (h * FNV_PRIME) & MASK64;
+    }
+    return h;
+}
+/**
+ * 64-bit SimHash over word shingles (unigrams + bigrams) of the normalized text. Near-identical posts
+ * (a swapped word, a different emoji, another handle) land within a few bits of each other.
+ */
+export function simhash64(normalized) {
+    const words = normalized.split(' ').filter(Boolean);
+    if (words.length === 0)
+        return 0n;
+    const shingles = [];
+    for (let i = 0; i < words.length; i++) {
+        shingles.push(words[i]);
+        if (i + 1 < words.length)
+            shingles.push(`${words[i]} ${words[i + 1]}`);
+    }
+    const v = new Array(64).fill(0);
+    for (const sh of shingles) {
+        const h = fnv1a64(sh);
+        for (let b = 0; b < 64; b++)
+            v[b] += (h >> BigInt(b)) & 1n ? 1 : -1;
+    }
+    let out = 0n;
+    for (let b = 0; b < 64; b++)
+        if (v[b] > 0)
+            out |= 1n << BigInt(b);
+    return out;
+}
+export function hamming(a, b) {
+    let x = a ^ b;
+    let n = 0;
+    while (x) {
+        x &= x - 1n;
+        n++;
+    }
+    return n;
+}
+/**
+ * Group near-duplicates (rule 1). Returns the same post objects with `cluster`, `dupCount` and
+ * `hidden` set: the earliest post of a cluster is the head (listed; carries dupCount), the rest hidden.
+ * Posts of different platforms or about different mints never cluster (a news headline is not a dupe
+ * of an X post quoting it). Idempotent.
+ */
+export function clusterPosts(posts, opts = {}) {
+    const windowMs = opts.windowMs ?? DEDUPE_WINDOW_MS;
+    const maxBits = opts.maxBits ?? DEDUPE_MAX_BITS;
+    const sorted = [...posts].sort((a, b) => a.t - b.t || a.id.localeCompare(b.id));
+    const heads = new Map(); // key: platform|mint
+    for (const p of sorted) {
+        const norm = normalizeForDedupe(p.text);
+        const hash = simhash64(norm);
+        const key = `${p.platform}|${p.mints[0] ?? ''}`;
+        const list = heads.get(key) ?? [];
+        let hit = null;
+        for (let i = list.length - 1; i >= 0; i--) {
+            const h = list[i];
+            if (p.t - h.t > windowMs)
+                break; // list is time-ordered; older heads cannot match
+            if (norm.length === 0)
+                continue; // empty text never clusters
+            if (h.hash === hash || hamming(h.hash, hash) <= maxBits) {
+                hit = h;
+                break;
+            }
+        }
+        if (hit) {
+            hit.members.push(p);
+            p.cluster = hit.id;
+            p.hidden = true;
+            p.dupCount = 1;
+        }
+        else {
+            list.push({ id: p.id, t: p.t, hash, members: [p] });
+            heads.set(key, list);
+            p.cluster = null;
+            p.hidden = false;
+            p.dupCount = 1;
+        }
+    }
+    for (const list of heads.values()) {
+        for (const h of list) {
+            if (h.members.length > 1) {
+                const head = h.members[0];
+                head.cluster = h.id;
+                head.dupCount = h.members.length;
+            }
+        }
+    }
+    return posts;
+}
+/**
+ * Rule 2, per mint: a sliding 2-minute window holding ≥ 10 posts from accounts younger than 30 days
+ * flags every post in it `spam`, demotes those authors to the `spam` tier and collapses the burst onto
+ * its earliest listed post. Accounts with an unknown creation time never count as fresh (a missing
+ * field is not evidence). Idempotent.
+ */
+export function markBurstSpam(posts, opts) {
+    const minPosts = opts.minPosts ?? BURST_MIN_POSTS;
+    const spanMs = opts.spanMs ?? BURST_SPAN_MS;
+    const freshMs = opts.freshAccountMs ?? FRESH_ACCOUNT_MS;
+    const byMint = new Map();
+    for (const p of posts) {
+        if (p.platform === 'news' || p.platform === 'chain')
+            continue;
+        const fresh = p.author.createdAt !== null && opts.now - p.author.createdAt <= freshMs;
+        if (!fresh)
+            continue;
+        for (const m of p.mints.length ? p.mints : ['']) {
+            const l = byMint.get(m) ?? [];
+            l.push(p);
+            byMint.set(m, l);
+        }
+    }
+    for (const list of byMint.values()) {
+        list.sort((a, b) => a.t - b.t);
+        let j = 0;
+        const flagged = new Set();
+        for (let i = 0; i < list.length; i++) {
+            while (list[i].t - list[j].t > spanMs)
+                j++;
+            if (i - j + 1 >= minPosts)
+                for (let k = j; k <= i; k++)
+                    flagged.add(list[k]);
+        }
+        if (flagged.size === 0)
+            continue;
+        const arr = [...flagged].sort((a, b) => a.t - b.t);
+        const accounts = new Set(arr.map((p) => p.author.id));
+        const spanSec = Math.max(1, Math.round((arr[arr.length - 1].t - arr[0].t) / 1000));
+        // collapse onto the earliest visible post of the burst (or the earliest post when all are hidden)
+        const head = arr.find((p) => !p.hidden) ?? arr[0];
+        for (const p of arr) {
+            if (!p.flags.includes('spam'))
+                p.flags.push('spam');
+            p.author.tier = 'spam';
+            if (p !== head)
+                p.hidden = true;
+        }
+        head.hidden = false;
+        head.cluster = head.cluster ?? head.id;
+        head.dupCount = Math.max(head.dupCount, arr.length);
+        head.collapsed = { posts: arr.length, accounts: accounts.size, spanSec };
+    }
+    return posts;
+}
+/** Rule 3: cashtag squats and unverified cashtags. Idempotent. */
+export function markSquats(posts, opts = {}) {
+    for (const p of posts) {
+        if (p.platform === 'news' || p.platform === 'chain')
+            continue;
+        const addrs = opts.addressesOf?.(p) ?? [];
+        const own = new Set(p.mints);
+        if (p.cashtags.length && addrs.some((a) => !own.has(a))) {
+            if (!p.flags.includes('squat'))
+                p.flags.push('squat');
+        }
+        if (p.mints.length === 0 && p.cashtags.length && opts.mintsForSymbol) {
+            const ambiguous = p.cashtags.some((s) => opts.mintsForSymbol(s).length >= SQUAT_MIN_LIVE_MINTS);
+            if (ambiguous && !p.flags.includes('unverified-cashtag'))
+                p.flags.push('unverified-cashtag');
+        }
+    }
+    return posts;
+}
+/** Posts that count: listed heads, not spam, not squats/unverified, not injection. */
+export function countsForVelocity(p) {
+    return !p.hidden && !p.flags.includes('spam') && !p.flags.includes('squat') && !p.flags.includes('unverified-cashtag') && !p.flags.includes('injection');
+}