@tangle-network/agent-knowledge 19.1.5 → 20.0.0

package/dist/sources/index.d.ts

@@ -1,376 +0,0 @@
-//#region src/sources/types.d.ts
-/**
- * Pluggable knowledge source contract.
- *
- * A `KnowledgeSource` is one external provider of authoritative content that
- * an agent's knowledge base should track over time (e.g. Cornell LII US Code,
- * IRS publications, a state secretary-of-state filing portal). It returns
- * hashable, embed-ready `KnowledgeFragment`s plus enough provenance metadata
- * for downstream consumers to:
- *
- *   1. detect change against a previous snapshot (see `./changes`)
- *   2. score freshness on a per-source-id basis (see `./freshness`)
- *   3. decide which evals to re-run when the underlying authority moves
- *      (the `dimensionHints` field is the binding contract for that decision)
- *
- * Sources MUST be pure with respect to local filesystem state outside the
- * cache directory the caller hands them — they read remote authorities and
- * return data. They MUST mark `verifiable: false` on any fragment they could
- * not fetch and extract (block page, 4xx, parse failure) rather than silently
- * substituting empty or partial content. The control loop downstream uses
- * `verifiable` to refuse promotion of unusable content.
- *
- * @stable
- */
-/**
- * Per-fetch options the host (control loop / cron / CLI) passes in.
- *
- * `signal` lets the host abort long-running fetches (rate-limited authority,
- * congested network). `cacheDir` is where the source SHOULD write its disk
- * cache; an undefined value disables caching (useful in tests). `now` is
- * injected for deterministic tests of change-detection windows.
- */
-export interface FetchOpts {
-  /** Abort signal forwarded to the underlying HTTP fetcher. */
-  signal?: AbortSignal;
-  /** Absolute path under which the source may cache raw bytes. */
-  cacheDir?: string;
-  /** Clock injection for deterministic tests. */
-  now?: () => Date;
-  /**
-   * Maximum number of authority pages the source should fetch in this call.
-   * Sources MUST respect this bound — exhaustively crawling Cornell LII on
-   * every cron tick would be both rude and slow. Default is source-specific.
-   */
-  limit?: number;
-  /**
-   * Source-specific selector string. Examples:
-   *   - cornell-lii: `'uscode/text/18/1836'` or `'wex/non-compete'`
-   *   - irs-publications: `'index'` or `'p15'`
-   *   - state-sos: opaque, see `StateSosSourceConfig`
-   *
-   * Sources that don't need a selector ignore this field.
-   */
-  selector?: string;
-}
-/**
- * The standard provenance shape every fragment carries. Kept separate from
- * `KnowledgeFragment` so freshness/change code can pass it around without
- * also dragging the body text.
- */
-export interface FragmentProvenance {
-  /** Canonical URL the fragment was extracted from. */
-  url: string;
-  /**
-   * Source-attested timestamp: the time the AUTHORITY last updated this
-   * content, as reported by the source (Last-Modified header, in-page
-   * effective date, registry generated-at, etc). Falls back to the fetch
-   * time only when the authority publishes no timestamp.
-   */
-  sourceUpdatedAt: string;
-  /** ISO timestamp the fragment was fetched. */
-  fetchedAt: string;
-  /**
-   * Jurisdiction the content is binding within, if applicable. Use ISO
-   * country code, US state abbreviation, or 'US-FED' for federal scope.
-   * Statute sources MUST populate this; reference / encyclopedia sources
-   * MAY leave it undefined.
-   */
-  jurisdiction?: string;
-  /**
-   * True iff the configured URL returned an acceptable response and the
-   * expected content was extracted. False on a block page, rate-limit
-   * response, 4xx/5xx, or selector miss. This is not publisher authentication
-   * or cryptographic content verification. Consumers MUST refuse to promote
-   * `verifiable: false` fragments into citable knowledge.
-   */
-  verifiable: boolean;
-  /** If `verifiable === false`, the reason — surfaced to operators. */
-  unverifiableReason?: string;
-}
-/**
- * One unit of authoritative content. Stable hash on `(id, body)` lets change
- * detection reason about identity across snapshots.
- */
-export interface KnowledgeFragment {
-  /**
-   * Stable identity within (sourceId, selector-space). Two fetches against
-   * the same authority section MUST produce the same `id`. The (sourceId,
-   * id) pair is the primary key for change detection.
-   */
-  id: string;
-  /** Free-form title — section heading, publication name, etc. */
-  title: string;
-  /** Body text, normalised: no HTML tags, line breaks preserved. */
-  body: string;
-  /** SHA-256 of `body`. Pre-computed so consumers don't re-hash on diff. */
-  bodyHash: string;
-  provenance: FragmentProvenance;
-  /**
-   * Eval dimensions an agent-eval campaign should re-score when this
-   * fragment changes. Examples: `citation_hygiene`, `jurisdictional_accuracy`,
-   * `tax_compliance`, `regulatory_currency`. The eval cron treats this as a
-   * set, not a contract — adding a new dimension is non-breaking.
-   *
-   * This is the load-bearing field for the continuous-ingestion story: a
-   * Ryan-LLC-style ruling vacates the FTC non-compete rule → the source
-   * returns a fragment with `jurisdictional_accuracy` in this list →
-   * `detectChanges()` emits a `KnowledgeChange` carrying that hint → the
-   * cron knows exactly which agent-eval campaigns to re-run.
-   */
-  dimensionHints: string[];
-  /** Arbitrary source-specific metadata for debugging / connector wiring. */
-  metadata?: Record<string, unknown>;
-}
-/**
- * One pluggable knowledge source.
- *
- * Implementations: see `./cornell-lii`, `./irs-publications`, `./state-sos`.
- * To author a new source, follow the same shape and register it in your
- * application's source list — there is no global registry by design (per
- * the per-tenant isolation contract; see README).
- */
-export interface KnowledgeSource {
-  /** Stable id — used to key freshness state. MUST NOT change once shipped. */
-  id: string;
-  /** Human-readable name for dashboards. */
-  name: string;
-  /** One-sentence description: what authority + scope. */
-  description: string;
-  /**
-   * Pull fragments for this source. Sources MUST:
-   *   - rate-limit themselves (>=1 req/sec per source by convention)
-   *   - send a polite User-Agent
-   *   - cache to disk when `opts.cacheDir` is set
-   *   - mark `verifiable: false` rather than throwing on parse/block
-   *   - honour `opts.signal`
-   *   - honour `opts.limit`
-   */
-  fetch(opts: FetchOpts): Promise<KnowledgeFragment[]>;
-}
-//#endregion
-//#region src/sources/cornell-lii.d.ts
-export interface CornellLiiSelector {
-  /** Either 'uscode' or 'wex'. */
-  kind: 'uscode' | 'wex';
-  /**
-   * For `uscode`: `<title>/<section>` (e.g. `'18/1836'` for DTSA).
-   * For `wex`: the slug (e.g. `'non-compete'`).
-   */
-  path: string;
-  /**
-   * Optional pre-declared eval dimensions affected by this section. If
-   * omitted, defaults are chosen from `kind` + path heuristics.
-   */
-  dimensionHints?: string[];
-}
-export interface CornellLiiSourceOptions {
-  /**
-   * Selectors to fetch on each `fetch()` call. The caller (a per-tenant
-   * workspace config, typically) lists exactly the authorities they need
-   * tracked. There is no auto-discovery; that would crawl Cornell at
-   * cron speed, which is what the polite-fetch contract exists to avoid.
-   */
-  selectors: CornellLiiSelector[];
-  /** Source id override; default is `'cornell-lii'`. */
-  id?: string;
-}
-/**
- * Build a Cornell LII source for the listed selectors.
- *
- * Example: track DTSA + non-compete:
- * ```
- * createCornellLiiSource({
- *   selectors: [
- *     { kind: 'uscode', path: '18/1836' },
- *     { kind: 'wex', path: 'non-compete', dimensionHints: ['jurisdictional_accuracy'] },
- *   ],
- * })
- * ```
- */
-export declare function createCornellLiiSource(options: CornellLiiSourceOptions): KnowledgeSource;
-//#endregion
-//#region src/sources/html.d.ts
-/**
- * Minimal HTML helpers used by the shipped sources.
- *
- * Deliberately not a full DOM parser: every authority we ship against
- * (Cornell LII, IRS.gov, state SOS portals) has well-behaved server-rendered
- * HTML where regex-based extraction is correct and cheap. Bringing in cheerio
- * would add a 1.5MB dependency to a package whose purpose is shipping
- * primitives, not parsing arbitrary web pages.
- *
- * If a future source needs real DOM traversal, it should depend on its own
- * parser locally rather than promoting one into the package-wide deps.
- *
- * @stable
- */
-/**
- * Strip HTML tags, collapse whitespace, decode common entities.
- *
- * Preserves paragraph and line breaks (`</p>`, `<br>`, `</li>`, `</div>`,
- * `</h*>`) as `\n` so statute text retains its subsection structure.
- */
-export declare function htmlToText(html: string): string;
-/** Extract the first match of a regex's first capture group, or undefined. */
-export declare function firstMatch(html: string, pattern: RegExp): string | undefined;
-/** Extract the inner HTML of the first matching tag with id `id`. */
-export declare function innerHtmlById(html: string, id: string): string | undefined;
-/**
- * Extract every (href, text) pair matching the URL regex.
- * Returns absolute URLs by resolving against `baseUrl`.
- */
-export declare function extractLinks(html: string, hrefPattern: RegExp, baseUrl: string): {
-  href: string;
-  text: string;
-}[];
-//#endregion
-//#region src/sources/http.d.ts
-/**
- * Polite HTTP fetcher shared by remote sources.
- *
- * Independent sources share a per-origin throttle because rate-limited sites
- * may return block pages instead of 429 responses. Responses are cached by URL
- * because many publishers omit reliable ETag and Last-Modified headers. Bodies
- * are checked even after a 2xx response because captcha and block pages often
- * use successful status codes.
- */
-/** User-Agent string sent on every outbound request. */
-export declare const POLITE_USER_AGENT = "agent-knowledge (+https://github.com/tangle-network/agent-knowledge)";
-/** Minimum gap between successive requests to the same origin (ms). */
-export declare const MIN_REQUEST_GAP_MS = 1000;
-/** Maximum response body we will buffer in memory (bytes). */
-export declare const MAX_RESPONSE_BYTES: number;
-export interface PoliteFetchOptions {
-  signal?: AbortSignal;
-  cacheDir?: string;
-  /**
-   * Cache age beyond which we re-fetch. Default 1 hour, long enough to
-   * batch a cron sweep across many selectors, short enough that hourly
-   * authoritative-page changes get picked up next tick.
-   */
-  cacheTtlMs?: number;
-  /**
-   * Extra request headers. The fetcher always sets `User-Agent` and
-   * `Accept`; callers can add `Accept-Language` etc.
-   */
-  headers?: Record<string, string>;
-}
-export interface PoliteFetchResult {
-  url: string;
-  status: number;
-  /** Decoded UTF-8 body. Truncated to `MAX_RESPONSE_BYTES`. */
-  body: string;
-  /**
-   * Best-effort source-attested timestamp. Reads `Last-Modified`,
-   * falling back to `Date`, falling back to fetch time. Always ISO 8601.
-   */
-  sourceUpdatedAt: string;
-  fetchedAt: string;
-  /** True iff the response was satisfied from disk cache. */
-  fromCache: boolean;
-  /**
-   * False on: non-2xx status, captcha/block page heuristic match, or
-   * decoded body below 200 chars from a host known to serve real content
-   * (Cornell, IRS, state SOS). `unverifiableReason` carries the why.
-   */
-  verifiable: boolean;
-  unverifiableReason?: string;
-}
-/**
- * Fetch one URL with per-host throttling, on-disk cache, and block-page
- * detection. Never throws on network/HTTP failure. It returns a result with
- * `verifiable: false` and `unverifiableReason` set so the caller can decide
- * whether to skip, retry, or surface.
- *
- * Throws ONLY on `AbortError` (caller asked to stop) and on cache-write
- * failures that indicate a misconfigured filesystem.
- */
-export declare function politeFetch(url: string, options?: PoliteFetchOptions): Promise<PoliteFetchResult>;
-/** Reset the in-process throttle map. Test-only. */
-export declare function __resetHttpThrottle(): void;
-/** Cheap heuristic that catches CAPTCHA, WAF block pages, and "Just a moment" interstitials. */
-export declare function looksLikeBlockPage(body: string): boolean;
-//#endregion
-//#region src/sources/irs-publications.d.ts
-export interface IrsPublicationsSourceOptions {
-  /**
-   * Specific publication slugs to fetch (e.g. `['p15', 'p17', 'p463']`).
-   * When `includeIndex` is true (default), the publications index page is
-   * also fetched as a single fragment so change detection can notice
-   * year/revision shifts across the whole catalogue.
-   */
-  publications?: string[];
-  /**
-   * Revenue procedure paths to fetch (e.g. `['/irb/2024-31_IRB']`). The
-   * caller passes the exact path; this source does not auto-discover.
-   */
-  revenueProcedures?: string[];
-  includeIndex?: boolean;
-  id?: string;
-}
-/** Default eval dimensions for IRS-sourced fragments. */
-export declare const IRS_DIMENSION_HINTS: string[];
-export declare function createIrsPublicationsSource(options?: IrsPublicationsSourceOptions): KnowledgeSource;
-//#endregion
-//#region src/sources/state-sos.d.ts
-/**
- * Generic Secretary-of-State source.
- *
- * Every US state SOS surfaces LLC/Corp formation requirements differently
- * (CA via static forms pages, DE via division of corporations pages, TX
- * via SOSDirect content pages). Rather than baking 50 state-specific
- * parsers into this package, the source takes a config that names the URL
- * pattern + CSS-equivalent selector + jurisdiction tag. Callers supply one
- * config per state they need tracked.
- *
- * The selector is interpreted as a substring/regex of an HTML element id
- * or class — see `StateSosSourceConfig` for the contract. This is
- * intentionally minimal; richer extraction belongs in a state-specific
- * adapter the consumer authors.
- *
- * @experimental Interface will likely grow as we add more state coverage.
- */
-export interface StateSosEntity {
-  /** Stable id for this fragment within the state (e.g. 'llc-formation', 'corp-formation'). */
-  id: string;
-  /** Path under the configured `baseUrl` for this entity. */
-  path: string;
-  /**
-   * Extraction selector. Choose one:
-   *   - `{ kind: 'id', value: 'main-content' }` — innermost match of element with that id
-   *   - `{ kind: 'class', value: 'field--name-body' }` — innermost match of element with that class
-   *   - `{ kind: 'regex', value: /<article[\s\S]*?<\/article>/i }` — raw regex
-   *   - `{ kind: 'whole' }` — full body, tags stripped (fallback for unstructured pages)
-   */
-  selector: {
-    kind: 'id';
-    value: string;
-  } | {
-    kind: 'class';
-    value: string;
-  } | {
-    kind: 'regex';
-    value: RegExp;
-  } | {
-    kind: 'whole';
-  };
-  title: string;
-  /** Eval dimensions this entity feeds. */
-  dimensionHints?: string[];
-}
-export interface StateSosSourceConfig {
-  /** US state postal code, e.g. 'CA', 'DE', 'TX'. */
-  state: string;
-  /** Base URL for the state SOS — e.g. 'https://www.sos.ca.gov'. */
-  baseUrl: string;
-  /** Entities this state exposes (LLC, Corp, etc). */
-  entities: StateSosEntity[];
-  /** Source id; default `state-sos:<state>`. */
-  id?: string;
-  /** Display name; default `<state> Secretary of State`. */
-  name?: string;
-}
-export declare function createStateSosSource(config: StateSosSourceConfig): KnowledgeSource;
-//#endregion
-//# sourceMappingURL=index.d.ts.map

package/dist/sources/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/sources/types.ts","../../src/sources/cornell-lii.ts","../../src/sources/html.ts","../../src/sources/http.ts","../../src/sources/irs-publications.ts","../../src/sources/state-sos.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAgCiB;;EAEf,SAAS;;EAET;;EAEA,YAAY;;;;;;EAMZ;;;;;;;;;EASA;;;;;;;iBAQe;;EAEf;;;;;;;EAOA;;EAEA;;;;;;;EAOA;;;;;;;;EAQA;;EAEA;;;;;;iBAOe;;;;;;EAMf;;EAEA;;EAEA;;EAEA;EACA,YAAY;;;;;;;;;;;;;EAaZ;;EAEA,WAAW;;;;;;;;;;iBAWI;;EAEf;;EAEA;;EAEA;;;;;;;;;;EAUA,MAAM,MAAM,YAAY,QAAQ;;;;iBCrIjB;;EAEf;;;;;EAKA;;;;;EAKA;;iBAGe;;;;;;;EAOf,WAAW;;EAEX;;;;;;;;;;;;;;;wBAgBc,uBAAuB,SAAS,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;wBCrC1D,WAAW;;wBA4BX,WAAW,cAAc,SAAS;;wBAKlC,cAAc,cAAc;;;;;wBAa5B,aACd,cACA,aAAa,QACb;EACG;EAAc;;;;;;;;;;;;;;qBCxDN;;qBAIA;;qBAGA;iBAII;EACf,SAAS;EACT;;;;;;EAMA;;;;;EAKA,UAAU;;iBAGK;EACf;EACA;;EAEA;;;;;EAKA;EACA;;EAEA;;;;;;EAMA;EACA;;;;;;;;;;;wBAYoB,YACpB,aACA,UAAS,qBACR,QAAQ;;wBAoEK;;wBA6DA,mBAAmB;;;iBChLlB;;;;;;;EAOf;;;;;EAKA;EACA;EACA;;;qBAIW;wBAEG,4BACd,UAAS,+BACR;;;;;;;;;;;;;;;;;;;;iBC5Bc;;EAEf;;EAEA;;;;;;;;EAQA;IACM;IAAY;;IACZ;IAAe;;IACf;IAAe,OAAO;;IACtB;;EACN;;EAEA;;iBAGe;;EAEf;;EAEA;;EAEA,UAAU;;EAEV;;EAEA;;wBAGc,qBAAqB,QAAQ,uBAAuB"}