@tangle-network/agent-knowledge 1.2.0 → 1.3.0

package/dist/sources/index.d.ts

@@ -0,0 +1,386 @@
+/**
+ * Pluggable knowledge source contract.
+ *
+ * A `KnowledgeSource` is one external provider of authoritative content that
+ * an agent's knowledge base should track over time (e.g. Cornell LII US Code,
+ * IRS publications, a state secretary-of-state filing portal). It returns
+ * hashable, embed-ready `KnowledgeFragment`s plus enough provenance metadata
+ * for downstream consumers to:
+ *
+ *   1. detect change against a previous snapshot (see `./changes`)
+ *   2. score freshness on a per-source-id basis (see `./freshness`)
+ *   3. decide which evals to re-run when the underlying authority moves
+ *      (the `dimensionHints` field is the binding contract for that decision)
+ *
+ * Sources MUST be pure with respect to local filesystem state outside the
+ * cache directory the caller hands them — they read remote authorities and
+ * return data. They MUST mark `verifiable: false` on any fragment they could
+ * not authenticate (block page, 4xx, parse failure) rather than silently
+ * substituting empty/partial content. The control loop downstream uses
+ * `verifiable` to refuse promotion of un-grounded content.
+ *
+ * @stable
+ */
+/**
+ * Per-fetch options the host (control loop / cron / CLI) passes in.
+ *
+ * `signal` lets the host abort long-running fetches (rate-limited authority,
+ * congested network). `cacheDir` is where the source SHOULD write its disk
+ * cache; an undefined value disables caching (useful in tests). `now` is
+ * injected for deterministic tests of change-detection windows.
+ */
+interface FetchOpts {
+    /** Abort signal forwarded to the underlying HTTP fetcher. */
+    signal?: AbortSignal;
+    /** Absolute path under which the source may cache raw bytes. */
+    cacheDir?: string;
+    /** Clock injection for deterministic tests. */
+    now?: () => Date;
+    /**
+     * Maximum number of authority pages the source should fetch in this call.
+     * Sources MUST respect this bound — exhaustively crawling Cornell LII on
+     * every cron tick would be both rude and slow. Default is source-specific.
+     */
+    limit?: number;
+    /**
+     * Source-specific selector string. Examples:
+     *   - cornell-lii: `'uscode/text/18/1836'` or `'wex/non-compete'`
+     *   - irs-publications: `'index'` or `'p15'`
+     *   - state-sos: opaque, see `StateSosSourceConfig`
+     *
+     * Sources that don't need a selector ignore this field.
+     */
+    selector?: string;
+}
+/**
+ * The standard provenance shape every fragment carries. Kept separate from
+ * `KnowledgeFragment` so freshness/change code can pass it around without
+ * also dragging the body text.
+ */
+interface FragmentProvenance {
+    /** Canonical URL the fragment was extracted from. */
+    url: string;
+    /**
+     * Source-attested timestamp: the time the AUTHORITY last updated this
+     * content, as reported by the source (Last-Modified header, in-page
+     * effective date, registry generated-at, etc). Falls back to the fetch
+     * time only when the authority publishes no timestamp.
+     */
+    sourceUpdatedAt: string;
+    /** ISO timestamp the fragment was fetched. */
+    fetchedAt: string;
+    /**
+     * Jurisdiction the content is binding within, if applicable. Use ISO
+     * country code, US state abbreviation, or 'US-FED' for federal scope.
+     * Statute sources MUST populate this; reference / encyclopedia sources
+     * MAY leave it undefined.
+     */
+    jurisdiction?: string;
+    /**
+     * True iff the source could authenticate the fetched content (HTTP 200,
+     * expected selectors present, parse succeeded). False on any block page,
+     * rate-limit response, 4xx/5xx, or selector miss. Consumers MUST refuse
+     * to promote `verifiable: false` fragments into citable knowledge.
+     */
+    verifiable: boolean;
+    /** If `verifiable === false`, the reason — surfaced to operators. */
+    unverifiableReason?: string;
+}
+/**
+ * One unit of authoritative content. Stable hash on `(id, body)` lets change
+ * detection reason about identity across snapshots.
+ */
+interface KnowledgeFragment {
+    /**
+     * Stable identity within (sourceId, selector-space). Two fetches against
+     * the same authority section MUST produce the same `id`. The (sourceId,
+     * id) pair is the primary key for change detection.
+     */
+    id: string;
+    /** Free-form title — section heading, publication name, etc. */
+    title: string;
+    /** Body text, normalised: no HTML tags, line breaks preserved. */
+    body: string;
+    /** SHA-256 of `body`. Pre-computed so consumers don't re-hash on diff. */
+    bodyHash: string;
+    provenance: FragmentProvenance;
+    /**
+     * Eval dimensions an agent-eval campaign should re-score when this
+     * fragment changes. Examples: `citation_hygiene`, `jurisdictional_accuracy`,
+     * `tax_compliance`, `regulatory_currency`. The eval cron treats this as a
+     * set, not a contract — adding a new dimension is non-breaking.
+     *
+     * This is the load-bearing field for the continuous-ingestion story: a
+     * Ryan-LLC-style ruling vacates the FTC non-compete rule → the source
+     * returns a fragment with `jurisdictional_accuracy` in this list →
+     * `detectChanges()` emits a `KnowledgeChange` carrying that hint → the
+     * cron knows exactly which agent-eval campaigns to re-run.
+     */
+    dimensionHints: string[];
+    /** Arbitrary source-specific metadata for debugging / connector wiring. */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * One pluggable knowledge source.
+ *
+ * Implementations: see `./cornell-lii`, `./irs-publications`, `./state-sos`.
+ * To author a new source, follow the same shape and register it in your
+ * application's source list — there is no global registry by design (per
+ * the per-tenant isolation contract; see README).
+ */
+interface KnowledgeSource {
+    /** Stable id — used to key freshness state. MUST NOT change once shipped. */
+    id: string;
+    /** Human-readable name for dashboards. */
+    name: string;
+    /** One-sentence description: what authority + scope. */
+    description: string;
+    /**
+     * Pull fragments for this source. Sources MUST:
+     *   - rate-limit themselves (>=1 req/sec per source by convention)
+     *   - send a polite User-Agent
+     *   - cache to disk when `opts.cacheDir` is set
+     *   - mark `verifiable: false` rather than throwing on parse/block
+     *   - honour `opts.signal`
+     *   - honour `opts.limit`
+     */
+    fetch(opts: FetchOpts): Promise<KnowledgeFragment[]>;
+}
+
+interface CornellLiiSelector {
+    /** Either 'uscode' or 'wex'. */
+    kind: 'uscode' | 'wex';
+    /**
+     * For `uscode`: `<title>/<section>` (e.g. `'18/1836'` for DTSA).
+     * For `wex`: the slug (e.g. `'non-compete'`).
+     */
+    path: string;
+    /**
+     * Optional pre-declared eval dimensions affected by this section. If
+     * omitted, defaults are chosen from `kind` + path heuristics.
+     */
+    dimensionHints?: string[];
+}
+interface CornellLiiSourceOptions {
+    /**
+     * Selectors to fetch on each `fetch()` call. The caller (a per-tenant
+     * workspace config, typically) lists exactly the authorities they need
+     * tracked. There is no auto-discovery; that would crawl Cornell at
+     * cron speed, which is what the polite-fetch contract exists to avoid.
+     */
+    selectors: CornellLiiSelector[];
+    /** Source id override; default is `'cornell-lii'`. */
+    id?: string;
+}
+/**
+ * Build a Cornell LII source for the listed selectors.
+ *
+ * Example: track DTSA + non-compete:
+ * ```
+ * createCornellLiiSource({
+ *   selectors: [
+ *     { kind: 'uscode', path: '18/1836' },
+ *     { kind: 'wex', path: 'non-compete', dimensionHints: ['jurisdictional_accuracy'] },
+ *   ],
+ * })
+ * ```
+ */
+declare function createCornellLiiSource(options: CornellLiiSourceOptions): KnowledgeSource;
+
+/**
+ * Minimal HTML helpers used by the shipped sources.
+ *
+ * Deliberately not a full DOM parser: every authority we ship against
+ * (Cornell LII, IRS.gov, state SOS portals) has well-behaved server-rendered
+ * HTML where regex-based extraction is correct and cheap. Bringing in cheerio
+ * would add a 1.5MB dependency to a package whose purpose is shipping
+ * primitives, not parsing arbitrary web pages.
+ *
+ * If a future source needs real DOM traversal, it should depend on its own
+ * parser locally rather than promoting one into the package-wide deps.
+ *
+ * @stable
+ */
+/**
+ * Strip HTML tags, collapse whitespace, decode common entities.
+ *
+ * Preserves paragraph and line breaks (`</p>`, `<br>`, `</li>`, `</div>`,
+ * `</h*>`) as `\n` so statute text retains its subsection structure.
+ */
+declare function htmlToText(html: string): string;
+/** Extract the first match of a regex's first capture group, or undefined. */
+declare function firstMatch(html: string, pattern: RegExp): string | undefined;
+/** Extract the inner HTML of the first matching tag with id `id`. */
+declare function innerHtmlById(html: string, id: string): string | undefined;
+/**
+ * Extract every (href, text) pair matching the URL regex.
+ * Returns absolute URLs by resolving against `baseUrl`.
+ */
+declare function extractLinks(html: string, hrefPattern: RegExp, baseUrl: string): {
+    href: string;
+    text: string;
+}[];
+
+/**
+ * Polite HTTP fetcher used by every shipped source.
+ *
+ * Three invariants this enforces — each was a bug found while wiring real
+ * authorities; do not regress:
+ *
+ *   1. Per-host throttling. Cornell LII serves under 1 req/s/origin
+ *      politely and will start serving block pages above that. The lock
+ *      is per-host (`hostThrottle`) rather than per-source so that two
+ *      independent sources targeting the same authority still cooperate.
+ *
+ *   2. On-disk content cache keyed by URL. Production sources are called
+ *      from a cron loop; without a cache, every run re-hits the same
+ *      pages and inflates change-detection false-positives (the authority
+ *      occasionally serves slightly different boilerplate). The cache is
+ *      content-addressed by URL, not by ETag — authorities like IRS.gov
+ *      do not consistently send ETag/Last-Modified.
+ *
+ *   3. Block-page detection on success. A 200 with a captcha body still
+ *      means "we couldn't authenticate." Sources downstream rely on
+ *      `verifiable` to refuse promotion — losing that signal because the
+ *      fetcher said "well, the status code was 200" is the bug class
+ *      this exists to prevent.
+ *
+ * @stable
+ */
+/** User-Agent string sent on every outbound request. */
+declare const POLITE_USER_AGENT = "agent-knowledge/0.2.0 (+https://github.com/tangle-network/agent-knowledge)";
+/** Minimum gap between successive requests to the same origin (ms). */
+declare const MIN_REQUEST_GAP_MS = 1000;
+/** Maximum response body we will buffer in memory (bytes). */
+declare const MAX_RESPONSE_BYTES: number;
+interface PoliteFetchOptions {
+    signal?: AbortSignal;
+    cacheDir?: string;
+    /**
+     * Cache age beyond which we re-fetch. Default 1 hour — long enough to
+     * batch a cron sweep across many selectors, short enough that hourly
+     * authoritative-page changes get picked up next tick.
+     */
+    cacheTtlMs?: number;
+    /**
+     * Extra request headers. The fetcher always sets `User-Agent` and
+     * `Accept`; callers can add `Accept-Language` etc.
+     */
+    headers?: Record<string, string>;
+}
+interface PoliteFetchResult {
+    url: string;
+    status: number;
+    /** Decoded UTF-8 body. Truncated to `MAX_RESPONSE_BYTES`. */
+    body: string;
+    /**
+     * Best-effort source-attested timestamp. Reads `Last-Modified`,
+     * falling back to `Date`, falling back to fetch time. Always ISO 8601.
+     */
+    sourceUpdatedAt: string;
+    fetchedAt: string;
+    /** True iff the response was satisfied from disk cache. */
+    fromCache: boolean;
+    /**
+     * False on: non-2xx status, captcha/block page heuristic match, or
+     * decoded body below 200 chars from a host known to serve real content
+     * (Cornell, IRS, state SOS). `unverifiableReason` carries the why.
+     */
+    verifiable: boolean;
+    unverifiableReason?: string;
+}
+/**
+ * Fetch one URL with per-host throttling, on-disk cache, and block-page
+ * detection. Never throws on network/HTTP failure — returns a result with
+ * `verifiable: false` and `unverifiableReason` set so the caller can decide
+ * whether to skip, retry, or surface.
+ *
+ * Throws ONLY on `AbortError` (caller asked to stop) and on cache-write
+ * failures that indicate a misconfigured filesystem.
+ */
+declare function politeFetch(url: string, options?: PoliteFetchOptions): Promise<PoliteFetchResult>;
+/** Reset the in-process throttle map. Test-only. */
+declare function __resetHttpThrottle(): void;
+/** Cheap heuristic that catches CAPTCHA, WAF block pages, and "Just a moment" interstitials. */
+declare function looksLikeBlockPage(body: string): boolean;
+
+interface IrsPublicationsSourceOptions {
+    /**
+     * Specific publication slugs to fetch (e.g. `['p15', 'p17', 'p463']`).
+     * When `includeIndex` is true (default), the publications index page is
+     * also fetched as a single fragment so change detection can notice
+     * year/revision shifts across the whole catalogue.
+     */
+    publications?: string[];
+    /**
+     * Revenue procedure paths to fetch (e.g. `['/irb/2024-31_IRB']`). The
+     * caller passes the exact path; this source does not auto-discover.
+     */
+    revenueProcedures?: string[];
+    includeIndex?: boolean;
+    id?: string;
+}
+/** Default eval dimensions for IRS-sourced fragments. */
+declare const IRS_DIMENSION_HINTS: string[];
+declare function createIrsPublicationsSource(options?: IrsPublicationsSourceOptions): KnowledgeSource;
+
+/**
+ * Generic Secretary-of-State source.
+ *
+ * Every US state SOS surfaces LLC/Corp formation requirements differently
+ * (CA via static forms pages, DE via division of corporations pages, TX
+ * via SOSDirect content pages). Rather than baking 50 state-specific
+ * parsers into this package, the source takes a config that names the URL
+ * pattern + CSS-equivalent selector + jurisdiction tag. Callers supply one
+ * config per state they need tracked.
+ *
+ * The selector is interpreted as a substring/regex of an HTML element id
+ * or class — see `StateSosSourceConfig` for the contract. This is
+ * intentionally minimal; richer extraction belongs in a state-specific
+ * adapter the consumer authors.
+ *
+ * @experimental Interface will likely grow as we add more state coverage.
+ */
+interface StateSosEntity {
+    /** Stable id for this fragment within the state (e.g. 'llc-formation', 'corp-formation'). */
+    id: string;
+    /** Path under the configured `baseUrl` for this entity. */
+    path: string;
+    /**
+     * Extraction selector. Choose one:
+     *   - `{ kind: 'id', value: 'main-content' }` — innermost match of element with that id
+     *   - `{ kind: 'class', value: 'field--name-body' }` — innermost match of element with that class
+     *   - `{ kind: 'regex', value: /<article[\s\S]*?<\/article>/i }` — raw regex
+     *   - `{ kind: 'whole' }` — full body, tags stripped (fallback for unstructured pages)
+     */
+    selector: {
+        kind: 'id';
+        value: string;
+    } | {
+        kind: 'class';
+        value: string;
+    } | {
+        kind: 'regex';
+        value: RegExp;
+    } | {
+        kind: 'whole';
+    };
+    title: string;
+    /** Eval dimensions this entity feeds. */
+    dimensionHints?: string[];
+}
+interface StateSosSourceConfig {
+    /** US state postal code, e.g. 'CA', 'DE', 'TX'. */
+    state: string;
+    /** Base URL for the state SOS — e.g. 'https://www.sos.ca.gov'. */
+    baseUrl: string;
+    /** Entities this state exposes (LLC, Corp, etc). */
+    entities: StateSosEntity[];
+    /** Source id; default `state-sos:<state>`. */
+    id?: string;
+    /** Display name; default `<state> Secretary of State`. */
+    name?: string;
+}
+declare function createStateSosSource(config: StateSosSourceConfig): KnowledgeSource;
+
+export { type CornellLiiSelector, type CornellLiiSourceOptions, type FetchOpts, type FragmentProvenance, IRS_DIMENSION_HINTS, type IrsPublicationsSourceOptions, type KnowledgeFragment, type KnowledgeSource, MAX_RESPONSE_BYTES, MIN_REQUEST_GAP_MS, POLITE_USER_AGENT, type PoliteFetchOptions, type PoliteFetchResult, type StateSosEntity, type StateSosSourceConfig, __resetHttpThrottle, createCornellLiiSource, createIrsPublicationsSource, createStateSosSource, extractLinks, firstMatch, htmlToText, innerHtmlById, looksLikeBlockPage, politeFetch };

package/dist/sources/index.js

@@ -0,0 +1,34 @@
+import {
+  IRS_DIMENSION_HINTS,
+  MAX_RESPONSE_BYTES,
+  MIN_REQUEST_GAP_MS,
+  POLITE_USER_AGENT,
+  __resetHttpThrottle,
+  createCornellLiiSource,
+  createIrsPublicationsSource,
+  createStateSosSource,
+  extractLinks,
+  firstMatch,
+  htmlToText,
+  innerHtmlById,
+  looksLikeBlockPage,
+  politeFetch
+} from "../chunk-WCYW2GDA.js";
+import "../chunk-YMKHCTS2.js";
+export {
+  IRS_DIMENSION_HINTS,
+  MAX_RESPONSE_BYTES,
+  MIN_REQUEST_GAP_MS,
+  POLITE_USER_AGENT,
+  __resetHttpThrottle,
+  createCornellLiiSource,
+  createIrsPublicationsSource,
+  createStateSosSource,
+  extractLinks,
+  firstMatch,
+  htmlToText,
+  innerHtmlById,
+  looksLikeBlockPage,
+  politeFetch
+};
+//# sourceMappingURL=index.js.map

package/dist/sources/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/{types-DTUp66Gr.d.ts → types-CAeh7Lwb.d.ts}

@@ -170,4 +170,4 @@ interface KnowledgeRelease {
     metadata?: Record<string, unknown>;
 }
 
-export type { ClaimRef as C, KnowledgeGraphEdge as K, SourceRecord as S, KnowledgeGraphNode as a, KnowledgeGraph as b, KnowledgeWriteParseResult as c, KnowledgeEventType as d, KnowledgeEvent as e, KnowledgePage as f, KnowledgeIndex as g, SourceRegistry as h, KnowledgeSearchResult as i, KnowledgeLintFinding as j, KnowledgeBaseCandidate as k, KnowledgeRelease as l, KnowledgeClaim as m, KnowledgeId as n, KnowledgePolicy as o, KnowledgeRelation as p, KnowledgeUnit as q, KnowledgeWriteBlock as r, SourceAnchor as s };
+export type { ClaimRef as C, KnowledgeGraphEdge as K, SourceRecord as S, KnowledgeGraphNode as a, KnowledgeGraph as b, KnowledgeIndex as c, KnowledgeSearchResult as d, KnowledgeEventType as e, KnowledgeEvent as f, KnowledgePage as g, KnowledgeLintFinding as h, KnowledgeBaseCandidate as i, KnowledgeWriteBlock as j, KnowledgeClaim as k, KnowledgeRelease as l, SourceRegistry as m, KnowledgeWriteParseResult as n, KnowledgeId as o, KnowledgePolicy as p, KnowledgeRelation as q, KnowledgeUnit as r, SourceAnchor as s };

package/dist/viz/index.d.ts

@@ -1,4 +1,4 @@
-import { K as KnowledgeGraphEdge, a as KnowledgeGraphNode, b as KnowledgeGraph } from '../types-DTUp66Gr.js';
+import { K as KnowledgeGraphEdge, a as KnowledgeGraphNode, b as KnowledgeGraph } from '../types-CAeh7Lwb.js';
 
 interface KnowledgeVizNode extends KnowledgeGraphNode {
     degree: number;

package/dist/viz/index.js

@@ -2,7 +2,7 @@ import {
   detectKnowledgeGaps,
   findSurprisingConnections,
   toKnowledgeVizGraph
-} from "../chunk-TXNYP4WI.js";
+} from "../chunk-4PNXQ2NT.js";
 export {
   detectKnowledgeGaps,
   findSurprisingConnections,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-knowledge",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Source-grounded, eval-gated knowledge growth primitives for agents.",
   "homepage": "https://github.com/tangle-network/agent-knowledge#readme",
   "repository": {
@@ -28,6 +28,11 @@
       "types": "./dist/cli.d.ts",
       "import": "./dist/cli.js",
       "default": "./dist/cli.js"
+    },
+    "./sources": {
+      "types": "./dist/sources/index.d.ts",
+      "import": "./dist/sources/index.js",
+      "default": "./dist/sources/index.js"
     }
   },
   "bin": {
@@ -42,19 +47,12 @@
   "publishConfig": {
     "access": "public"
   },
-  "scripts": {
-    "build": "tsup",
-    "dev": "tsup --watch",
-    "prepare": "tsup",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "typecheck": "tsc --noEmit"
-  },
   "dependencies": {
-    "@tangle-network/agent-eval": "^0.20.0",
+    "@tangle-network/agent-eval": "^0.29.1",
     "zod": "^4.3.6"
   },
   "devDependencies": {
+    "@biomejs/biome": "^2.4.15",
     "@types/node": "^25.6.0",
     "tsup": "^8.0.0",
     "typescript": "^5.7.0",
@@ -64,5 +62,13 @@
     "node": ">=20"
   },
   "license": "MIT",
-  "packageManager": "pnpm@10.28.0"
-}
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit",
+    "lint": "biome check src tests",
+    "format": "biome format --write src tests"
+  }
+}