@neurowire/ingest 0.2.0 → 0.4.0

package/dist/index.d.ts

@@ -11,9 +11,29 @@ interface RawDocument {
     url: string;
     contentType: string;
     body: string;
+    etag?: string;
+    lastModified?: string;
+    /** True when the body was served from the cache via a 304 Not Modified. */
+    notModified?: boolean;
 }
+/** A previously fetched response plus its validators, kept for conditional requests. */
+interface CachedResponse {
+    url: string;
+    contentType: string;
+    body: string;
+    etag?: string;
+    lastModified?: string;
+}
+/** A store of cached responses, injected by the caller (the library keeps no global state). */
+interface ConditionalCache {
+    get(url: string): CachedResponse | undefined;
+    set(url: string, value: CachedResponse): void;
+}
+/** A simple Map-backed ConditionalCache. */
+declare function createMemoryCache(): ConditionalCache;
 interface FetchOptions {
     signal?: AbortSignal;
+    cache?: ConditionalCache;
 }
 /** Fetch a URL over HTTP(S), following redirects, returning the body and final URL. */
 declare function fetchDocument(url: string, options?: FetchOptions): Promise<RawDocument>;
@@ -92,6 +112,8 @@ interface FetchFeedOptions {
     signal?: AbortSignal;
     /** Max number of feed-link redirects to follow (default 3). */
     maxDepth?: number;
+    /** A conditional (ETag/Last-Modified) response cache, owned by the caller. */
+    cache?: ConditionalCache;
 }
 /** Fetch a URL (website, RSS, or Atom) and normalize it to a NeurowireFeed. */
 declare function fetchFeed(url: string, options?: FetchFeedOptions): Promise<NeurowireFeed>;
@@ -102,6 +124,8 @@ interface FetchMeshOptions {
     signal?: AbortSignal;
     /** Keep only the newest N merged entries. */
     limit?: number;
+    /** A conditional response cache shared by every mesh source. */
+    cache?: ConditionalCache;
 }
 /**
  * Fetch every source in a mesh (in parallel) and merge them into one feed.
@@ -124,10 +148,27 @@ declare function discoverFeedLink($: CheerioAPI, base: string): string | undefin
  */
 declare function autodetect($: CheerioAPI, ctx: ParseContext): NeurowireFeed | null;
 
+/** A proposed tap plus a preview of what it extracts, so a user can author a tap without DOM-spelunking. */
+interface TemplateProposal {
+    template: FeedTemplate;
+    matched: number;
+    sampleTitles: string[];
+}
+/**
+ * Inspect a feed-less HTML page and PROPOSE a FeedTemplate (CSS selectors) for it.
+ *
+ * Heuristic: find repeated item-like containers (sibling `article`/`li`/class-patterned
+ * `div`s that each hold a heading and an `<a href>`), pick the selector whose matched set
+ * is largest and consistent, then derive `title`/`link`/`date` selectors relative to the
+ * item. The candidate is validated by running `applyTemplate`: a proposal is returned only
+ * when it extracts at least one entry, otherwise `undefined`.
+ */
+declare function proposeTemplate(html: string, url: string): TemplateProposal | undefined;
+
 /** Register a per-host template. Validated with zod; ignored if it has no `host`. */
 declare function registerTemplate(template: FeedTemplate): void;
 /** Look up a template for a URL's hostname. */
 declare function findTemplate(url: string): FeedTemplate | undefined;
 declare function listTemplates(): FeedTemplate[];
 
-export { type FeedDraft, type FeedKind, type FeedTemplate, FeedTemplateSchema, type FetchFeedOptions, type FetchMeshOptions, type FetchOptions, type ParseContext, type RawDocument, applyTemplate, autodetect, detectKind, discoverFeedLink, fetchDocument, fetchFeed, fetchMesh, finalizeFeed, findTemplate, ingestDocument, listTemplates, normDate, parseAtom, parseFeedString, parseJsonFeed, parseRdf, parseRss, registerTemplate, resolveUrl, stripHtml };
+export { type CachedResponse, type ConditionalCache, type FeedDraft, type FeedKind, type FeedTemplate, FeedTemplateSchema, type FetchFeedOptions, type FetchMeshOptions, type FetchOptions, type ParseContext, type RawDocument, type TemplateProposal, applyTemplate, autodetect, createMemoryCache, detectKind, discoverFeedLink, fetchDocument, fetchFeed, fetchMesh, finalizeFeed, findTemplate, ingestDocument, listTemplates, normDate, parseAtom, parseFeedString, parseJsonFeed, parseRdf, parseRss, proposeTemplate, registerTemplate, resolveUrl, stripHtml };

package/dist/index.js

@@ -16,6 +16,15 @@ function detectKind(contentType, body) {
 }
 
 // src/fetch.ts
+function createMemoryCache() {
+  const store = /* @__PURE__ */ new Map();
+  return {
+    get: (url) => store.get(url),
+    set: (url, value) => {
+      store.set(url, value);
+    }
+  };
+}
 var USER_AGENT = "Neurowire/0.1 (+https://github.com/neurowire/neurowire)";
 var ACCEPT = "application/atom+xml, application/rss+xml, application/feed+json, application/json;q=0.9, text/html;q=0.8, */*;q=0.5";
 async function fetchDocument(url, options = {}) {
@@ -28,17 +37,37 @@ async function fetchDocument(url, options = {}) {
   if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
     throw new Error(`Unsupported protocol: ${parsed.protocol}`);
   }
+  const headers = { "user-agent": USER_AGENT, accept: ACCEPT };
+  const cached = options.cache?.get(url);
+  if (cached?.etag) headers["if-none-match"] = cached.etag;
+  if (cached?.lastModified) headers["if-modified-since"] = cached.lastModified;
   const res = await fetch(url, {
     redirect: "follow",
     signal: options.signal,
-    headers: { "user-agent": USER_AGENT, accept: ACCEPT }
+    headers
   });
+  if (res.status === 304 && cached) {
+    return {
+      url: cached.url,
+      contentType: cached.contentType,
+      body: cached.body,
+      etag: cached.etag,
+      lastModified: cached.lastModified,
+      notModified: true
+    };
+  }
   if (!res.ok) {
     throw new Error(`Upstream responded ${res.status} ${res.statusText} for ${url}`);
   }
   const contentType = res.headers.get("content-type") ?? "";
   const body = await res.text();
-  return { url: res.url || url, contentType, body };
+  const etag = res.headers.get("etag") ?? void 0;
+  const lastModified = res.headers.get("last-modified") ?? void 0;
+  const finalUrl = res.url || url;
+  if (options.cache) {
+    options.cache.set(url, { url: finalUrl, contentType, body, etag, lastModified });
+  }
+  return { url: finalUrl, contentType, body, etag, lastModified };
 }
 
 // src/ingest.ts
@@ -96,7 +125,10 @@ function attr(node, name) {
 }
 
 // src/util.ts
-import { GENERATOR } from "@neurowire/core";
+import {
+  GENERATOR,
+  stableId
+} from "@neurowire/core";
 function resolveUrl(href, base) {
   try {
     return new URL(href, base).toString();
@@ -124,13 +156,17 @@ function newestEntryDate(entries) {
   }
   return max === Number.NEGATIVE_INFINITY ? void 0 : new Date(max).toISOString();
 }
+function withStableId(entry) {
+  if (entry.id.trim()) return entry;
+  return { ...entry, id: stableId(entry.link, entry.title) };
+}
 function finalizeFeed(draft, ctx) {
   const updated = normDate(draft.updated) ?? newestEntryDate(draft.entries) ?? (/* @__PURE__ */ new Date()).toISOString();
   const feed = {
     id: draft.id?.trim() || ctx.sourceUrl,
     title: draft.title?.trim() || "Untitled",
     updated,
-    entries: draft.entries,
+    entries: draft.entries.map(withStableId),
     generator: { name: GENERATOR.name, version: GENERATOR.version }
   };
   if (draft.home) feed.home = resolveUrl(draft.home, ctx.sourceUrl);
@@ -209,7 +245,7 @@ function jsonLdEntry(node, ctx) {
   const title = str(node.headline) ?? str(node.name);
   if (!url || !title) return null;
   const link = resolveUrl(url, ctx.sourceUrl);
-  const entry = { id: link, title, link };
+  const entry = { id: "", title, link };
   const published = normDate(str(node.datePublished));
   const updated = normDate(str(node.dateModified));
   if (published) entry.published = published;
@@ -258,7 +294,7 @@ function fromSemantic($, ctx) {
     const href = $a.attr("href") ?? $el.find("a[href]").first().attr("href");
     if (!title || !href) return;
     const link = resolveUrl(href, ctx.sourceUrl);
-    const entry = { id: link, title, link };
+    const entry = { id: "", title, link };
     const $time = $el.find("time[datetime]").first();
     const date = normDate($time.attr("datetime") ?? $el.find("time").first().text().trim());
     if (date) entry.published = date;
@@ -322,7 +358,7 @@ function applyTemplate($, template, ctx) {
     const href = $link.attr("href") ?? $link.find("a").first().attr("href");
     if (!title2 || !href) return;
     const link = resolveUrl(href, ctx.sourceUrl);
-    const entry = { id: link, title: title2, link };
+    const entry = { id: "", title: title2, link };
     if (template.date) {
       const $date = $el.find(template.date).first();
       const date = normDate($date.attr("datetime") ?? $date.text().trim());
@@ -403,7 +439,7 @@ function atomEntry(node, ctx) {
   const href = pickLink(links, "alternate") ?? attr(links[0], "href") ?? text(get(node, "id")) ?? "";
   const link = resolveUrl(href, ctx.sourceUrl);
   const entry = {
-    id: text(get(node, "id")) ?? link,
+    id: text(get(node, "id")) ?? "",
     title: text(get(node, "title")) ?? "Untitled",
     link
   };
@@ -439,7 +475,7 @@ function rssItem(node, ctx) {
   const guid = text(get(node, "guid"));
   const link = resolveUrl(text(get(node, "link")) ?? guid ?? "", ctx.sourceUrl);
   const entry = {
-    id: guid ?? link,
+    id: guid ?? "",
     title: text(get(node, "title")) ?? "Untitled",
     link
   };
@@ -497,7 +533,7 @@ function parseJsonFeed(raw, ctx) {
   const entries = toArray(data.items).map((item) => {
     const link = resolveUrl(item.url ?? item.external_url ?? "", ctx.sourceUrl);
     const entry = {
-      id: item.id !== void 0 ? String(item.id) : link,
+      id: item.id !== void 0 ? String(item.id) : "",
       title: item.title ?? "Untitled",
       link
     };
@@ -538,7 +574,7 @@ async function fetchFeed(url, options = {}) {
   return ingest(url, options, 0);
 }
 async function ingest(url, options, depth) {
-  const doc = await fetchDocument(url, { signal: options.signal });
+  const doc = await fetchDocument(url, { signal: options.signal, cache: options.cache });
   return ingestDocument(doc, options, depth);
 }
 async function ingestDocument(doc, options = {}, depth = 0) {
@@ -571,7 +607,7 @@ async function ingestDocument(doc, options = {}, depth = 0) {
 // src/mesh.ts
 import { mergeFeeds } from "@neurowire/core";
 async function fetchMesh(mesh, options = {}) {
-  const fetchOptions = { signal: options.signal };
+  const fetchOptions = { signal: options.signal, cache: options.cache };
   const results = await Promise.allSettled(
     mesh.sources.map(
       async (source) => ({
@@ -589,10 +625,116 @@ async function fetchMesh(mesh, options = {}) {
   }
   return mergeFeeds(mesh.name, parts, { limit: options.limit });
 }
+
+// src/html/propose.ts
+import { load as load2 } from "cheerio";
+var HEADING_SELECTOR = "h1, h2, h3, h4";
+var TITLE_CANDIDATES = ["h2", "h3", "h1", "h4"];
+function hostOf(url) {
+  try {
+    return new URL(url).hostname || void 0;
+  } catch {
+    return void 0;
+  }
+}
+function looksLikeItem($, el) {
+  const $el = $(el);
+  const hasLink = $el.is("a[href]") || $el.find("a[href]").length > 0;
+  const hasTitle = $el.find(HEADING_SELECTOR).length > 0 || $el.is("a[href]");
+  return hasLink && hasTitle;
+}
+function selectorFor($, el) {
+  const $el = $(el);
+  const tag = ($el.prop("tagName") ?? "div").toLowerCase();
+  const className = ($el.attr("class") ?? "").trim();
+  if (!className) return tag;
+  const first = className.split(/\s+/)[0];
+  return first ? `${tag}.${first}` : tag;
+}
+function candidateSelectors($) {
+  const counts = /* @__PURE__ */ new Map();
+  $("article, li, div, a[href]").each((_, el) => {
+    if (!looksLikeItem($, el)) return;
+    const selector = selectorFor($, el);
+    counts.set(selector, (counts.get(selector) ?? 0) + 1);
+  });
+  for (const base of ["article"]) {
+    const n = $(base).filter((_, el) => looksLikeItem($, el)).length;
+    if (n > 0) counts.set(base, Math.max(counts.get(base) ?? 0, n));
+  }
+  const anchorRooted = (selector) => selector === "a" || selector.startsWith("a.");
+  return [...counts.entries()].filter(([, n]) => n >= 2).sort((a, b) => b[1] - a[1] || Number(anchorRooted(a[0])) - Number(anchorRooted(b[0]))).map(([selector]) => selector);
+}
+function titleSelectorFor($, itemSelector) {
+  const $first = $(itemSelector).first();
+  if ($first.is("a[href]") && !$first.find(HEADING_SELECTOR).length) return "a";
+  for (const candidate of TITLE_CANDIDATES) {
+    if ($first.find(candidate).first().text().trim()) return candidate;
+  }
+  if ($first.find("a[href]").first().text().trim()) return "a";
+  return void 0;
+}
+function itemIsLink($, itemSelector) {
+  let total = 0;
+  let anchors = 0;
+  $(itemSelector).each((_, el) => {
+    total += 1;
+    if ($(el).is("a[href]")) anchors += 1;
+  });
+  return total > 0 && anchors === total;
+}
+function linkSelectorFor($, itemSelector, titleSelector) {
+  if (itemIsLink($, itemSelector)) return void 0;
+  const $first = $(itemSelector).first();
+  if ($first.find(`${titleSelector} a[href]`).length) return `${titleSelector} a`;
+  return "a";
+}
+function dateSelectorFor($, itemSelector) {
+  const $first = $(itemSelector).first();
+  if ($first.find("time").length) return "time";
+  if ($first.find("[datetime]").length) return "[datetime]";
+  return void 0;
+}
+function feedTitleOf($) {
+  const title = $("title").first().text().trim();
+  if (title) return title;
+  const h1 = $("h1").first().text().trim();
+  return h1 || void 0;
+}
+function proposeTemplate(html, url) {
+  const $ = load2(html);
+  const ctx = { sourceUrl: url };
+  let best;
+  for (const itemSelector of candidateSelectors($)) {
+    const titleSelector = titleSelectorFor($, itemSelector);
+    if (!titleSelector) continue;
+    const template = { item: itemSelector, title: titleSelector };
+    const host = hostOf(url);
+    if (host) template.host = host;
+    const feedTitle = feedTitleOf($);
+    if (feedTitle) template.feedTitle = feedTitle;
+    const link = linkSelectorFor($, itemSelector, titleSelector);
+    if (link) template.link = link;
+    const date = dateSelectorFor($, itemSelector);
+    if (date) template.date = date;
+    const feed = applyTemplate($, template, ctx);
+    const matched = feed.entries.length;
+    if (matched === 0) continue;
+    if (!best || matched > best.matched) {
+      best = {
+        template,
+        matched,
+        sampleTitles: feed.entries.slice(0, 5).map((e) => e.title)
+      };
+    }
+  }
+  return best;
+}
 export {
   FeedTemplateSchema,
   applyTemplate,
   autodetect,
+  createMemoryCache,
   detectKind,
   discoverFeedLink,
   fetchDocument,
@@ -608,6 +750,7 @@ export {
   parseJsonFeed,
   parseRdf,
   parseRss,
+  proposeTemplate,
   registerTemplate,
   resolveUrl,
   stripHtml

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@neurowire/ingest",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "Fetch, detect, and parse RSS/Atom/RDF/JSON feeds and HTML pages into the Neurowire model.",
   "type": "module",
   "main": "./dist/index.js",
@@ -48,7 +48,7 @@
     "cheerio": "^1.0.0",
     "fast-xml-parser": "^4.5.1",
     "zod": "^3.24.1",
-    "@neurowire/core": "0.3.0"
+    "@neurowire/core": "0.5.0"
   },
   "devDependencies": {
     "@types/node": "^22.10.5",