@aitne/daemon 0.1.8 → 0.1.9

package/dist/services/browser-history/pipeline/project-matcher.js

@@ -0,0 +1,314 @@
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { basename, join } from "node:path";
+/**
+ * WEEKLY_INTERESTS_REFLECTION_PLAN.md §10.5 — frontmatter-driven
+ * keyword extraction for project files + per-project Jaccard / filename
+ * matcher.
+ *
+ * Pure helpers. The frontmatter parser here is a minimal flat-scalar
+ * reader with two extensions over `core/context-frontmatter.ts`:
+ *
+ *   1. Keys may contain `.` (e.g. `aitne.exclude_from_interests`) — the
+ *      existing flat parser is restricted to `[A-Za-z0-9_-]` keys.
+ *   2. Values may be YAML flow arrays (`[a, b, "c d"]`) or block
+ *      sequences (`\n  - a\n  - b\n`), both of which `aliases` / `tags`
+ *      may legitimately use.
+ *
+ * Everything else is identical to the rest of the codebase's
+ * frontmatter expectations: `---` fences delimit the block, the body
+ * follows. No multi-line scalar support; we don't need it for the
+ * three keys (`aitne_project_keywords`, `aitne.exclude_from_interests`,
+ * `aliases`, `tags`) we read.
+ */
+// CONTEXT_VAULT_REDESIGN: project files live under plans/projects/.
+const PROJECTS_DIR = "plans/projects";
+const KEY_PROJECT_KEYWORDS = "aitne_project_keywords";
+const KEY_EXCLUDE = "aitne.exclude_from_interests";
+const KEY_ALIASES = "aliases";
+const KEY_TAGS = "tags";
+const TOKEN_SPLIT = /[-_/\s]+/;
+const JACCARD_MIN_OVERLAP_TOKENS = 2;
+const JACCARD_MIN_RATIO = 0.15;
+const DEFAULT_MAX_CLUSTER_MATCHES = 5;
+function parseFrontmatter(content) {
+    const lines = content.split(/\r?\n/);
+    if (lines.length === 0 || lines[0].trim() !== "---")
+        return null;
+    const endIdx = lines.findIndex((line, index) => index > 0 && line.trim() === "---");
+    if (endIdx < 0)
+        return null;
+    const values = new Map();
+    let i = 1;
+    while (i < endIdx) {
+        const raw = lines[i];
+        const line = raw.trim();
+        if (line === "" || line.startsWith("#")) {
+            i += 1;
+            continue;
+        }
+        // Allow dots in keys (`aitne.exclude_from_interests:`).
+        const match = /^([A-Za-z0-9_.-]+)\s*:\s*(.*)$/.exec(line);
+        if (!match) {
+            i += 1;
+            continue;
+        }
+        const key = match[1];
+        const rest = match[2].trim();
+        if (rest === "" || rest === "[]") {
+            // Look ahead for a block sequence:
+            //   tags:
+            //     - foo
+            //     - bar
+            const items = [];
+            let j = i + 1;
+            while (j < endIdx) {
+                const next = lines[j];
+                const dashMatch = /^\s*-\s+(.*)$/.exec(next);
+                if (!dashMatch)
+                    break;
+                items.push(stripQuotes(dashMatch[1].trim()));
+                j += 1;
+            }
+            if (items.length > 0) {
+                values.set(key, items);
+                i = j;
+                continue;
+            }
+            values.set(key, "");
+            i += 1;
+            continue;
+        }
+        if (rest.startsWith("[") && rest.endsWith("]")) {
+            values.set(key, parseFlowArray(rest));
+            i += 1;
+            continue;
+        }
+        values.set(key, stripQuotes(rest));
+        i += 1;
+    }
+    return { values };
+}
+function parseFlowArray(value) {
+    // `value` is the full `[…]` text. Strip the brackets and split on
+    // commas, respecting quoted strings.
+    const inner = value.slice(1, -1);
+    const out = [];
+    let buf = "";
+    let quote = null;
+    for (let i = 0; i < inner.length; i++) {
+        const ch = inner[i];
+        if (quote) {
+            if (ch === quote) {
+                quote = null;
+            }
+            else {
+                buf += ch;
+            }
+            continue;
+        }
+        if (ch === '"' || ch === "'") {
+            quote = ch;
+            continue;
+        }
+        if (ch === ",") {
+            const trimmed = buf.trim();
+            if (trimmed.length > 0)
+                out.push(trimmed);
+            buf = "";
+            continue;
+        }
+        buf += ch;
+    }
+    const tail = buf.trim();
+    if (tail.length > 0)
+        out.push(tail);
+    return out;
+}
+// Quoted-scalar matcher: double- or single-quotes surrounding the
+// whole value. Anything else passes through unchanged.
+const QUOTED_SCALAR_RE = /^(?:"([^"]*)"|'([^']*)')$/;
+function stripQuotes(value) {
+    const m = QUOTED_SCALAR_RE.exec(value);
+    if (!m)
+        return value;
+    // Exactly one of the alternation groups is populated when the regex
+    // matches — both are bounded by `^…$` so they cannot be optional.
+    return (m[1] ?? m[2]);
+}
+function tokenize(value) {
+    return value
+        .toLowerCase()
+        .split(TOKEN_SPLIT)
+        .filter((tok) => tok.length > 1);
+}
+function isTruthyYamlBool(value) {
+    if (Array.isArray(value))
+        return false;
+    const v = value.trim().toLowerCase();
+    return v === "true" || v === "yes" || v === "on";
+}
+function asStringArray(value) {
+    if (Array.isArray(value))
+        return value;
+    // Scalar provided where an array was expected (e.g. `tags: foo`).
+    // Treat the whole scalar as a single entry the tokenizer will split.
+    return value.trim().length > 0 ? [value] : [];
+}
+/**
+ * Extract the keyword set for a single project file. Pure with
+ * respect to the inputs `content` and `filename`; the FS reader below
+ * (`loadProjectKeywords`) is the only impure entry.
+ *
+ * Returns `null` when the project opts out via
+ * `aitne.exclude_from_interests: true`.
+ */
+export function extractKeywordsFromFile(filename, content, projectPath) {
+    const projectSlug = filename.replace(/\.md$/i, "");
+    const fm = parseFrontmatter(content);
+    if (fm) {
+        const exclude = fm.values.get(KEY_EXCLUDE);
+        if (exclude !== undefined && isTruthyYamlBool(exclude))
+            return null;
+    }
+    // 1. Explicit override.
+    const explicit = fm?.values.get(KEY_PROJECT_KEYWORDS);
+    if (explicit !== undefined) {
+        const items = asStringArray(explicit);
+        const tokens = new Set();
+        for (const item of items) {
+            for (const tok of tokenize(item))
+                tokens.add(tok);
+        }
+        if (tokens.size > 0) {
+            return {
+                projectSlug,
+                projectPath,
+                keywords: tokens,
+                source: "explicit",
+            };
+        }
+        // Fall through if the override was an empty array — same as
+        // omitting the key.
+    }
+    // 2. Frontmatter aliases/tags + filename.
+    const tokens = new Set();
+    let frontmatterContributed = false;
+    if (fm) {
+        for (const key of [KEY_ALIASES, KEY_TAGS]) {
+            const value = fm.values.get(key);
+            if (value === undefined)
+                continue;
+            const items = asStringArray(value);
+            for (const item of items) {
+                for (const tok of tokenize(item)) {
+                    tokens.add(tok);
+                    frontmatterContributed = true;
+                }
+            }
+        }
+    }
+    // 3. Filename fallback (always added — it's free and never wrong).
+    for (const tok of tokenize(projectSlug))
+        tokens.add(tok);
+    return {
+        projectSlug,
+        projectPath,
+        keywords: tokens,
+        source: frontmatterContributed ? "frontmatter" : "filename",
+    };
+}
+/**
+ * Read every `*.md` directly under `<contextDir>/projects/`. Sub-
+ * directories are ignored — projects are flat by convention in this
+ * codebase. Missing or non-directory paths return `[]` quietly so the
+ * caller does not need to feature-detect.
+ */
+export function loadProjectKeywords(contextDir) {
+    const dir = join(contextDir, PROJECTS_DIR);
+    let entries;
+    try {
+        const stat = statSync(dir);
+        if (!stat.isDirectory())
+            return [];
+        entries = readdirSync(dir);
+    }
+    catch {
+        return [];
+    }
+    const out = [];
+    for (const entry of entries) {
+        if (!entry.toLowerCase().endsWith(".md"))
+            continue;
+        const filePath = join(dir, entry);
+        let stat;
+        try {
+            stat = statSync(filePath);
+        }
+        catch {
+            continue;
+        }
+        if (!stat.isFile())
+            continue;
+        let content;
+        try {
+            content = readFileSync(filePath, "utf-8");
+        }
+        catch {
+            continue;
+        }
+        const extracted = extractKeywordsFromFile(basename(entry), content, filePath);
+        if (extracted)
+            out.push(extracted);
+    }
+    out.sort((a, b) => a.projectSlug.localeCompare(b.projectSlug));
+    return out;
+}
+/**
+ * Single-project matcher. Pure. Returns up to `cap` matches sorted in
+ * the order they appear in `clusters` — the caller pre-sorts that
+ * input (the Step 1 builder ranks by foreground_sec desc).
+ */
+export function matchClustersToProject(project, clusters, cap = DEFAULT_MAX_CLUSTER_MATCHES) {
+    const matches = [];
+    const seen = new Set();
+    const projectSlugLower = project.projectSlug.toLowerCase();
+    for (const cluster of clusters) {
+        if (matches.length >= cap)
+            break;
+        if (cluster.displayName.toLowerCase().includes(projectSlugLower)) {
+            if (!seen.has(cluster.slug)) {
+                matches.push({ slug: cluster.slug, reason: "filename_match" });
+                seen.add(cluster.slug);
+            }
+            continue;
+        }
+        const clusterTokens = new Set();
+        for (const tok of tokenize(cluster.displayName))
+            clusterTokens.add(tok);
+        for (const domain of cluster.topDomains) {
+            const prefix = domain
+                .replace(/^www\./, "")
+                .split(".")[0]
+                .toLowerCase();
+            if (prefix.length > 1)
+                clusterTokens.add(prefix);
+        }
+        let overlap = 0;
+        for (const tok of project.keywords) {
+            if (clusterTokens.has(tok))
+                overlap += 1;
+        }
+        if (overlap < JACCARD_MIN_OVERLAP_TOKENS)
+            continue;
+        // Union size = |A| + |B| - |A ∩ B|. Avoids building the merged set.
+        const union = project.keywords.size + clusterTokens.size - overlap;
+        const ratio = overlap / union;
+        if (ratio < JACCARD_MIN_RATIO)
+            continue;
+        if (!seen.has(cluster.slug)) {
+            matches.push({ slug: cluster.slug, reason: "jaccard" });
+            seen.add(cluster.slug);
+        }
+    }
+    return matches;
+}

package/dist/services/browser-history/pipeline/redactor.d.ts

@@ -0,0 +1,15 @@
+export interface RawVisitInput {
+    url: string;
+    title: string | null;
+    searchQuery: string | null;
+}
+export interface RedactedVisit {
+    url: string;
+    scheme: string;
+    host: string;
+    path: string;
+    searchQuery: string | null;
+    title: string | null;
+    drop: boolean;
+}
+export declare function redactVisit(input: RawVisitInput): RedactedVisit;

package/dist/services/browser-history/pipeline/redactor.js

@@ -0,0 +1,119 @@
+import { classifyHost } from "./sensitive-hosts.js";
+const STRIPPED_QUERY_KEYS = new Set([
+    "orderid",
+    "order_id",
+    "order-id",
+    "token",
+    "access_token",
+    "id_token",
+    "refresh_token",
+    "auth_token",
+    "session",
+    "sessid",
+    "sessionid",
+    "session_id",
+    "secret",
+    "apikey",
+    "api_key",
+    "key",
+    "code",
+    "state",
+    "nonce",
+    "x-amz-security-token",
+]);
+const JWT_REGEX = /^[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}$/;
+const SECRET_TITLE_REGEX = /(api[_ -]?key|password|secret|bearer|access[_ -]?token)/i;
+const MAX_TITLE_LENGTH = 240;
+function stripSensitiveQueryParams(searchParams) {
+    const keys = [...searchParams.keys()];
+    for (const rawKey of keys) {
+        const key = rawKey.toLowerCase();
+        if (STRIPPED_QUERY_KEYS.has(key)) {
+            searchParams.delete(rawKey);
+            continue;
+        }
+        const value = searchParams.get(rawKey);
+        if (value && JWT_REGEX.test(value)) {
+            searchParams.delete(rawKey);
+        }
+    }
+}
+function safeTitle(title) {
+    if (!title)
+        return null;
+    if (SECRET_TITLE_REGEX.test(title))
+        return null;
+    const trimmed = title.trim();
+    if (trimmed.length === 0)
+        return null;
+    return trimmed.length > MAX_TITLE_LENGTH
+        ? trimmed.slice(0, MAX_TITLE_LENGTH)
+        : trimmed;
+}
+function safeSearchQuery(searchQuery) {
+    if (!searchQuery)
+        return null;
+    if (SECRET_TITLE_REGEX.test(searchQuery))
+        return null;
+    const trimmed = searchQuery.trim();
+    if (trimmed.length === 0)
+        return null;
+    return trimmed.length > MAX_TITLE_LENGTH
+        ? trimmed.slice(0, MAX_TITLE_LENGTH)
+        : trimmed;
+}
+export function redactVisit(input) {
+    let parsed;
+    try {
+        parsed = new URL(input.url);
+    }
+    catch {
+        return {
+            url: input.url,
+            scheme: "",
+            host: "",
+            path: "",
+            searchQuery: null,
+            title: null,
+            drop: true,
+        };
+    }
+    const scheme = parsed.protocol.toLowerCase();
+    const host = parsed.hostname.toLowerCase();
+    const sensitive = classifyHost(host);
+    if (sensitive === "adult") {
+        return {
+            url: "",
+            scheme,
+            host,
+            path: parsed.pathname,
+            searchQuery: null,
+            title: null,
+            drop: true,
+        };
+    }
+    stripSensitiveQueryParams(parsed.searchParams);
+    // Force a stable serialised form so the hash is deterministic.
+    parsed.hash = "";
+    const rebuilt = parsed.toString();
+    if (sensitive === "banking" || sensitive === "health") {
+        return {
+            url: rebuilt,
+            scheme,
+            host,
+            path: parsed.pathname,
+            searchQuery: null,
+            title: null,
+            drop: false,
+        };
+    }
+    return {
+        url: rebuilt,
+        scheme,
+        host,
+        path: parsed.pathname,
+        searchQuery: safeSearchQuery(input.searchQuery),
+        title: safeTitle(input.title),
+        drop: false,
+    };
+}

package/dist/services/browser-history/pipeline/reload-detector.d.ts

@@ -0,0 +1,6 @@
+export declare function isReloadTransition(transition: number | null | undefined): boolean;
+export interface ReloadPatternInput {
+    host: string;
+    path: string;
+}
+export declare function reloadPatternKey(input: ReloadPatternInput): string;

package/dist/services/browser-history/pipeline/reload-detector.js

@@ -0,0 +1,16 @@
+const PAGE_TRANSITION_CORE_MASK = 0xff;
+const PAGE_TRANSITION_RELOAD = 8;
+export function isReloadTransition(transition) {
+    if (transition === null || transition === undefined)
+        return false;
+    return (transition & PAGE_TRANSITION_CORE_MASK) === PAGE_TRANSITION_RELOAD;
+}
+export function reloadPatternKey(input) {
+    const segments = (input.path || "/")
+        .split("/")
+        .filter((segment) => segment.length > 0);
+    const firstSegment = segments[0] ?? "";
+    return firstSegment
+        ? `${input.host}/${firstSegment.toLowerCase()}`
+        : input.host;
+}

package/dist/services/browser-history/pipeline/sensitive-hosts.d.ts

@@ -0,0 +1,2 @@
+export type SensitiveHostCategory = "banking" | "health" | "adult";
+export declare function classifyHost(host: string): SensitiveHostCategory | null;

package/dist/services/browser-history/pipeline/sensitive-hosts.js

@@ -0,0 +1,173 @@
+const BANKING_SUFFIXES = [
+    "chase.com",
+    "bankofamerica.com",
+    "wellsfargo.com",
+    "citi.com",
+    "citibank.com",
+    "usbank.com",
+    "capitalone.com",
+    "americanexpress.com",
+    "amex.com",
+    "discover.com",
+    "paypal.com",
+    "venmo.com",
+    "wise.com",
+    "revolut.com",
+    "n26.com",
+    "monzo.com",
+    "starlingbank.com",
+    "barclays.co.uk",
+    "hsbc.com",
+    "hsbc.co.uk",
+    "lloydsbank.com",
+    "natwest.com",
+    "santander.co.uk",
+    "santander.com",
+    "smbc.co.jp",
+    "mufg.jp",
+    "bk.mufg.jp",
+    "smbctrust.co.jp",
+    "shinseibank.com",
+    "sonybank.net",
+    "rakuten-bank.co.jp",
+    "japannetbank.co.jp",
+    "paypay-bank.co.jp",
+    "ufj.jp",
+    "mizuhobank.co.jp",
+    "yuchobank.jp",
+    "jp-bank.japanpost.jp",
+    "stripe.com",
+    "square.com",
+    "squareup.com",
+    "coinbase.com",
+    "binance.com",
+    "kraken.com",
+    "bitflyer.com",
+    "fidelity.com",
+    "vanguard.com",
+    "schwab.com",
+    "etrade.com",
+    "interactivebrokers.com",
+    "robinhood.com",
+    "wealthsimple.com",
+];
+const HEALTH_SUFFIXES = [
+    "mychart.com",
+    "kp.org",
+    "kaiserpermanente.org",
+    "cvs.com",
+    "walgreens.com",
+    "webmd.com",
+    "mayoclinic.org",
+    "clevelandclinic.org",
+    "nih.gov",
+    "cdc.gov",
+    "who.int",
+    "uptodate.com",
+    "epicmychart.com",
+    "anthem.com",
+    "aetna.com",
+    "cigna.com",
+    "unitedhealthcare.com",
+    "humana.com",
+    "23andme.com",
+    "ancestry.com",
+    "doxy.me",
+    "teladoc.com",
+    "amwell.com",
+    "betterhelp.com",
+    "talkspace.com",
+    "headspace.com",
+    "calm.com",
+    "myfitnesspal.com",
+    "fitbit.com",
+    "ouraring.com",
+    "withings.com",
+];
+const ADULT_SUFFIXES = [
+    "pornhub.com",
+    "xvideos.com",
+    "xnxx.com",
+    "redtube.com",
+    "youporn.com",
+    "xhamster.com",
+    "onlyfans.com",
+    "chaturbate.com",
+    "cam4.com",
+    "stripchat.com",
+    "spankbang.com",
+    "tnaflix.com",
+    "brazzers.com",
+    "playboy.com",
+    "adultfriendfinder.com",
+    "fansly.com",
+    "manyvids.com",
+    "clips4sale.com",
+    "fanza.co.jp",
+    "dmm.co.jp",
+];
+const BANKING_KEYWORDS = [
+    "bank",
+    "banco",
+    "banque",
+    "banking",
+    "credit-union",
+    "creditunion",
+    "brokerage",
+];
+const HEALTH_KEYWORDS = [
+    "health",
+    "clinic",
+    "hospital",
+    "medical",
+    "pharma",
+    "pharmacy",
+    "wellness",
+    "therapy",
+];
+function matchesSuffix(host, suffixes) {
+    const cleaned = host.replace(/^www\./i, "").toLowerCase();
+    return suffixes.some((suffix) => cleaned === suffix || cleaned.endsWith(`.${suffix}`));
+}
+function registeredDomain(host) {
+    const parts = host.split(".");
+    if (parts.length < 2)
+        return host;
+    // Treat two-level public suffixes (co.jp, co.uk, com.au, …) as one
+    // unit. This is intentionally a small list; the heuristic falls back
+    // to last-two-labels for anything else, which is enough for the
+    // intent here (we only need the *registered* label for keyword
+    // probing).
+    const twoLevelTlds = ["co.jp", "co.uk", "com.au", "com.br", "co.kr"];
+    const lastTwo = parts.slice(-2).join(".");
+    if (twoLevelTlds.includes(lastTwo) && parts.length >= 3) {
+        return parts[parts.length - 3];
+    }
+    return parts[parts.length - 2];
+}
+function keywordHit(host, keywords) {
+    const reg = registeredDomain(host);
+    return keywords.some((keyword) => reg.includes(keyword));
+}
+export function classifyHost(host) {
+    if (!host)
+        return null;
+    const cleaned = host.replace(/^www\./i, "").toLowerCase();
+    if (matchesSuffix(cleaned, ADULT_SUFFIXES))
+        return "adult";
+    if (matchesSuffix(cleaned, BANKING_SUFFIXES))
+        return "banking";
+    if (matchesSuffix(cleaned, HEALTH_SUFFIXES))
+        return "health";
+    // Heuristic keyword match against the registered domain label only —
+    // catches "somelocalbank.com" / "acme-credit-union.com" without
+    // false-positiving "creditcardrewards.org" (registered domain is
+    // `creditcardrewards`, no "credit-union" / "bank" exact match). The
+    // failure mode is "we lose signal", never "we leak sensitive
+    // content".
+    if (keywordHit(cleaned, BANKING_KEYWORDS))
+        return "banking";
+    if (keywordHit(cleaned, HEALTH_KEYWORDS))
+        return "health";
+    return null;
+}

package/dist/services/browser-history/pipeline/summarizer.d.ts

@@ -0,0 +1,52 @@
+import { type BrowserHistoryBrowserKey } from "@aitne/shared";
+import type { ChromiumVisitRow } from "../readers/chromium-reader.js";
+import { type ResearchDomainLists } from "./meaningful-filter.js";
+export interface SummarizeDayBoundary {
+    timezone: string | undefined;
+    dayBoundaryHour: number;
+}
+export interface ReloadIncrementEmission {
+    date: string;
+    urlPattern: string;
+    count: number;
+}
+export interface SummarizedVisit {
+    ts: number;
+    browser: BrowserHistoryBrowserKey;
+    profile: string;
+    urlHash: string;
+    domain: string;
+    category: string;
+    meaningful: 0 | 1;
+    dwellSec: number | null;
+    foregroundSec: number | null;
+    transition: number;
+    isReload: 0 | 1;
+    rootTaskId: number | null;
+    httpStatus: number | null;
+    title: string | null;
+    searchQuery: string | null;
+    amazonAsin: string | null;
+    amazonLocale: string | null;
+}
+export interface SummarizeInput {
+    browser: BrowserHistoryBrowserKey;
+    profile: string;
+    rows: readonly ChromiumVisitRow[];
+    boundary?: SummarizeDayBoundary;
+    /**
+     * BROWSER_HISTORY_INTEGRATION_PLAN §5.F1.meaningful rule 6 — user-curated
+     * domain allowlist / denylist. Both default to an empty `Set` (emergent
+     * mode). When the allowlist is non-empty, only visits whose eTLD+1 is
+     * in the allowlist are eligible as meaningful. The denylist always
+     * wins. See `meaningful-filter.ts` for the precedence rules.
+     */
+    researchDomainLists?: ResearchDomainLists;
+}
+export interface SummarizeResult {
+    visits: SummarizedVisit[];
+    reloadIncrements: ReloadIncrementEmission[];
+    highestTimestampMs: number;
+    dropped: number;
+}
+export declare function summarizeVisits(input: SummarizeInput): SummarizeResult;