@titan-design/brain 0.2.2 → 0.3.0

package/dist/web-extract-K4LTMRW2.js

@@ -0,0 +1,111 @@
+// src/services/web-extract.ts
+import { Readability } from "@mozilla/readability";
+import TurndownService from "turndown";
+import { JSDOM } from "jsdom";
+import { createHash } from "crypto";
+var TRACKING_PARAMS = /* @__PURE__ */ new Set([
+  "utm_source",
+  "utm_medium",
+  "utm_campaign",
+  "utm_term",
+  "utm_content",
+  "fbclid",
+  "gclid",
+  "ref",
+  "source",
+  "mc_cid",
+  "mc_eid"
+]);
+function normalizeUrl(rawUrl) {
+  const url = new URL(rawUrl);
+  url.hostname = url.hostname.toLowerCase();
+  for (const param of TRACKING_PARAMS) {
+    url.searchParams.delete(param);
+  }
+  url.searchParams.sort();
+  let result = url.toString();
+  if (result.endsWith("/") && url.pathname !== "/") {
+    result = result.slice(0, -1);
+  }
+  return result;
+}
+function extractMetadata(html) {
+  const dom = new JSDOM(html);
+  const doc = dom.window.document;
+  const getMeta = (selectors) => {
+    for (const sel of selectors) {
+      const el = doc.querySelector(sel);
+      if (el) return el.getAttribute("content");
+    }
+    return null;
+  };
+  return {
+    title: getMeta(['meta[property="og:title"]', 'meta[name="title"]']) ?? doc.title ?? null,
+    author: getMeta(['meta[name="author"]', 'meta[property="article:author"]']),
+    description: getMeta(['meta[property="og:description"]', 'meta[name="description"]']),
+    siteName: getMeta(['meta[property="og:site_name"]']),
+    publishedDate: getMeta(['meta[property="article:published_time"]', 'meta[name="date"]'])
+  };
+}
+function extractFromHtml(html, url) {
+  const dom = new JSDOM(html, { url });
+  const reader = new Readability(dom.window.document);
+  const article = reader.parse();
+  const turndown = new TurndownService({
+    headingStyle: "atx",
+    codeBlockStyle: "fenced"
+  });
+  const markdown = article?.content ? turndown.turndown(article.content) : "";
+  const metadata = extractMetadata(html);
+  if (article?.title) metadata.title = article.title;
+  if (article?.byline) metadata.author = article.byline;
+  if (article?.siteName) metadata.siteName = article.siteName;
+  const contentHash = createHash("sha256").update(markdown).digest("hex");
+  return {
+    markdown,
+    metadata,
+    normalizedUrl: normalizeUrl(url),
+    contentHash
+  };
+}
+var DEFAULT_OPTIONS = {
+  timeout: 1e4,
+  maxSize: 5 * 1024 * 1024,
+  userAgent: "brain/1.0"
+};
+async function fetchAndExtract(url, options) {
+  const opts = { ...DEFAULT_OPTIONS, ...options };
+  const parsed = new URL(url);
+  if (!["http:", "https:"].includes(parsed.protocol)) {
+    throw new Error(`Unsupported protocol: ${parsed.protocol} (only http/https)`);
+  }
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), opts.timeout);
+  try {
+    const response = await fetch(url, {
+      signal: controller.signal,
+      headers: { "User-Agent": opts.userAgent },
+      redirect: "follow"
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    }
+    const contentLength = response.headers.get("content-length");
+    if (contentLength && Number(contentLength) > opts.maxSize) {
+      throw new Error(`Content too large: ${contentLength} bytes (max ${opts.maxSize})`);
+    }
+    const html = await response.text();
+    if (html.length > opts.maxSize) {
+      throw new Error(`Content too large: ${html.length} bytes (max ${opts.maxSize})`);
+    }
+    return extractFromHtml(html, url);
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}
+export {
+  extractFromHtml,
+  extractMetadata,
+  fetchAndExtract,
+  normalizeUrl
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@titan-design/brain",
-  "version": "0.2.2",
+  "version": "0.3.0",
   "type": "module",
   "description": "Developer second brain with hybrid RAG search",
   "license": "MIT",
@@ -23,31 +23,41 @@
     "test:watch": "vitest",
     "typecheck": "tsc --noEmit",
     "lint": "eslint src __tests__ --no-error-on-unmatched-pattern",
+    "format": "prettier --write \"src/**/*.ts\" \"__tests__/**/*.ts\"",
+    "format:check": "prettier --check \"src/**/*.ts\" \"__tests__/**/*.ts\"",
+    "test:coverage": "vitest run --coverage",
     "postinstall": "node scripts/postinstall.js",
     "preuninstall": "node scripts/preuninstall.js"
   },
   "dependencies": {
-    "commander": "^13.1.0",
     "@commander-js/extra-typings": "^13.1.0",
+    "@mozilla/readability": "^0.6.0",
     "better-sqlite3": "^11.8.1",
-    "sqlite-vec": "^0.1.6",
-    "ollama": "^0.5.14",
+    "commander": "^13.1.0",
+    "env-paths": "^3.0.0",
     "gray-matter": "^4.0.3",
-    "env-paths": "^3.0.0"
+    "jsdom": "^28.1.0",
+    "ollama": "^0.5.14",
+    "sqlite-vec": "^0.1.6",
+    "turndown": "^7.2.2"
   },
   "optionalDependencies": {
     "@huggingface/transformers": "^3.3.3"
   },
   "devDependencies": {
-    "typescript": "^5.7.3",
-    "vitest": "^3.0.5",
-    "tsup": "^8.3.6",
+    "@eslint/js": "^9.20.0",
     "@types/better-sqlite3": "^7.6.13",
+    "@types/jsdom": "^27.0.0",
     "@types/node": "^22.13.4",
-    "tsx": "^4.19.2",
+    "@types/turndown": "^5.0.6",
+    "@vitest/coverage-v8": "^3.2.4",
     "eslint": "^9.20.0",
-    "@eslint/js": "^9.20.0",
-    "typescript-eslint": "^8.24.1"
+    "prettier": "^3.8.1",
+    "tsup": "^8.3.6",
+    "tsx": "^4.19.2",
+    "typescript": "^5.7.3",
+    "typescript-eslint": "^8.24.1",
+    "vitest": "^3.0.5"
   },
   "publishConfig": {
     "access": "public",

package/skill/SKILL.md

@@ -5,7 +5,7 @@ description: Search and manage your second brain knowledge base. Use when the us
 
 # Brain -- Knowledge Management
 
-A CLI for managing a developer second brain with hybrid BM25 + vector search over markdown notes.
+A CLI for managing a developer second brain with hybrid BM25 + vector search, LLM-powered memory extraction, and temporal intelligence over markdown notes.
 
 ## When to Use
 
@@ -13,6 +13,8 @@ A CLI for managing a developer second brain with hybrid BM25 + vector search ove
 - User wants to save something to their knowledge base
 - User asks about stale or outdated notes
 - User wants to check their knowledge base status
+- User wants to capture a quick thought or link
+- User asks about extracted memories or facts
 
 ## Commands
 
@@ -21,14 +23,26 @@ Use `--json` flag on all commands when processing output programmatically.
 | Command | Purpose | Example |
 |---------|---------|---------|
 | `brain search "<query>" --json` | Hybrid search (BM25 + vector) | `brain search "authentication patterns" --json --limit 5` |
-| `brain search "<query>" --json --expand` | Search with graph-connected notes | `brain search "auth" --json --expand` |
+| `brain search "<query>" --memories --json` | Search notes + extracted memories | `brain search "auth" --memories --json` |
+| `brain search "<query>" --rerank --json` | Search with cross-encoder reranking | `brain search "auth" --rerank --json` |
+| `brain search "<query>" --expand --json` | Search with graph-connected notes | `brain search "auth" --json --expand` |
 | `brain add <file>` | Add a note from file | `brain add ~/draft.md --type research --tier slow` |
 | `brain add --title "X" --type note` | Add from stdin | `echo "content" \| brain add --title "My Note" --type note` |
+| `brain quick "thought"` | Zero-friction capture to inbox | `brain quick "look into WebSockets vs SSE"` |
+| `brain inbox --json` | View inbox items | `brain inbox --status pending --json` |
+| `brain extract --all` | Extract memories from all notes | Requires Ollama running locally |
+| `brain extract --note <id>` | Extract memories from one note | `brain extract --note my-note-id` |
+| `brain memories list --json` | List active memories | `brain memories list --container default --json` |
+| `brain memories history <id>` | Show memory version chain | `brain memories history mem-abc123` |
+| `brain memories stats` | Memory count + expiry sweep | Shows active count, runs auto-forget |
+| `brain context <id> --json` | Note context (relations + memories) | `brain context my-note --json` |
+| `brain profile --format json` | Agent context profile | `brain profile --container default --format json` |
 | `brain status --json` | Database stats | Shows note count, embeddings, staleness |
 | `brain stale --json` | Notes needing review | `brain stale --tier slow --json` |
 | `brain index` | Re-index all notes | Only run when user asks -- this is slow |
-| `brain template <type>` | Output frontmatter template | `brain template research` |
 | `brain graph <note-id> --json` | Show note relations | `brain graph my-note --json` |
+| `brain doctor --json` | System health checks | Shows DB, embedder, LLM, inbox, stale status |
+| `brain doctor --fix` | Auto-repair issues | Pulls missing models, resets failed inbox |
 | `brain config get` | Show config | `brain config get` |
 
 ## Search Filters