@3sln/trove 0.0.2

package/packages/core/src/index.js

@@ -0,0 +1,104 @@
+// @3sln/trove/core — runtime-agnostic building blocks for a self-hostable drive.
+// Import the pieces directly, or use `createVfs` to wire a sensible default.
+
+export { TroveError, ErrorCode, wrapError, isRetryable, isOutOfSpace } from './errors.js';
+export { withRetry, withTimeout } from './retry.js';
+export * from './util.js';
+export * from './links.js';
+
+export { StorageBackend } from './storage/interface.js';
+export { MemoryStorage } from './storage/memory.js';
+export { FilesystemStorage } from './storage/filesystem.js';
+export { S3Storage } from './storage/s3.js';
+export { PrefixedStorage } from './storage/prefixed.js';
+
+export { MetadataStore } from './metadata/interface.js';
+export { MemoryStore } from './metadata/memory.js';
+export { SqliteStore } from './metadata/sqlite.js';
+
+export { CollectionService, CAPABILITIES } from './collections/index.js';
+
+export { SearchService } from './search/index.js';
+export { SearchTransformer, ParsingSearchTransformer, WorkersAiSearchTransformer, parseTagFilters, matchTagFilters } from './search/transformer.js';
+export { VectorStore, MemoryVectorStore, QdrantVectorStore } from './search/vectorStore.js';
+export { KeywordStore, MemoryKeywordStore } from './search/keywordStore.js';
+export { EmbeddingProvider, LocalHashEmbedding, HttpEmbedding } from './search/embeddings.js';
+// Durable local search: vectors via sqlite-vec, keywords via FTS5, both in the
+// SQLite file the metadata already lives in.
+export { SqliteVectorStore, SqliteKeywordStore, SEARCH_DB_KEY } from './search/sqliteStores.js';
+
+export { IndexerRegistry, textIndexer, chunkText } from './indexers/registry.js';
+export { PluginService, PackageStore, StoragePackageStore, PluginInstallStore, SqlitePluginInstallStore, MemoryPluginInstallStore, parsePluginPackage, capabilityList, ALL_CAPABILITIES, IndexerRuntime, InProcessIndexerRuntime, PluginIndexers, matchFromSelector } from './plugins/index.js';
+export { UploadManager, DEFAULT_PART_SIZE } from './uploads.js';
+export { Vfs, CONTENT_TYPES } from './vfs.js';
+export { IndexingCoordinator } from './indexing.js';
+// Work in flight (ephemeral) and standing problems (durable) — see the header of each.
+export { TaskRegistry } from './tasks.js';
+export { IssueRegistry, issueId } from './issues.js';
+// Reconcile a collection against what its store actually holds (changes made outside Trove).
+export { CollectionScanner } from './scan.js';
+export { normalizeContribution, clampContribution, clampTagValue, DEFAULT_CAPS } from './indexers/contribution.js';
+
+// Cloudflare Vectorize — first-class pluggable vector DB.
+export { VectorizeVectorStore } from './search/vectorize.js';
+
+// Server-side key/value store (subscriptions, inboxes, profiles).
+export { KeyValueStore, MemoryKV, SqliteKV } from './kv.js';
+export { SignedUrls, resolveUrlSecret, URL_PURPOSES } from './signedUrls.js';
+export { SqliteDatabase, SqliteProvider, LocalSqliteProvider, assertSafePluginSql, stripSqlLiterals } from './sqlite.js';
+// SQLite on Cloudflare D1, so a Worker deployment has a metadata store that exists.
+export { D1SqliteProvider } from './sqlite-d1.js';
+
+// Identity (BYO IdP — Cloudflare Access / Zero Trust / a proxy).
+export {
+  IdentityProvider, JwtIdentityProvider, HeaderIdentityProvider,
+  AnonymousIdentityProvider, principalFromClaims,
+  cloudflareAccess, accessHost,
+} from './identity/index.js';
+export { verifyJwt, decodeJwt, JwksClient, StaticJwks } from './identity/jwt.js';
+// Where an unauthenticated client is told to go — one answer for the whole drive.
+export {
+  protectedResourceMetadata, challengeHeaders, metadataUrl, publicOrigin,
+  normalizeServers, resolveAuthDiscovery, usableAuthServer, headerSafe,
+} from './identity/discovery.js';
+
+// Sidecar documents: conversations, tags, indexer facets (CRDT, cold-in-S3).
+export { SidecarService, SidecarStore, SidecarManager } from './sidecar/index.js';
+export * as sidecarOps from './sidecar/document.js';
+
+// Notifications: mention batching + web push.
+export { NotificationCenter } from './notifications/index.js';
+export { WebPushService, generateVapidKeys } from './notifications/webpush.js';
+
+import { Vfs } from './vfs.js';
+import { MemoryStorage } from './storage/memory.js';
+import { MemoryStore } from './metadata/memory.js';
+import { SearchService } from './search/index.js';
+import { LocalHashEmbedding } from './search/embeddings.js';
+import { IndexerRegistry } from './indexers/registry.js';
+
+/**
+ * Wire a Vfs from parts, defaulting to in-memory everything (great for tests and
+ * a zero-config first run). Pass real backends for production:
+ *
+ *   createVfs({
+ *     storage: new S3Storage({ ... }),
+ *     metadata: new SqliteStore({ provider: new LocalSqliteProvider({ path: 'trove.db' }) }),
+ *     embeddings: new HttpEmbedding({ url, apiKey, model, dimensions: 1536 }),
+ *   })
+ *
+ * @param {object} [opts]
+ * @returns {Promise<Vfs>} initialised (metadata.init already run)
+ */
+export async function createVfs(opts = {}) {
+  const storage = opts.storage ?? new MemoryStorage();
+  const metadata = opts.metadata ?? new MemoryStore();
+  const embeddings = opts.embeddings ?? new LocalHashEmbedding();
+  const search =
+    opts.search ??
+    new SearchService({ embeddings, vectorStore: opts.vectorStore, keywordStore: opts.keywordStore });
+  const indexers = opts.indexers ?? new IndexerRegistry();
+  const vfs = new Vfs({ storage, metadata, search, indexers, sidecar: opts.sidecar, collections: opts.collections, searchTransformer: opts.searchTransformer, issues: opts.issues, maxIndexBytes: opts.maxIndexBytes, maxUploadBytes: opts.maxUploadBytes, uploadPartSize: opts.uploadPartSize });
+  await vfs.init();
+  return vfs;
+}

package/packages/core/src/indexers/contribution.js

@@ -0,0 +1,115 @@
+// What a *contribution* is, and how big it may be.
+//
+// A contribution is the one shape everything that enriches an item produces — the
+// built-in indexers, a plugin's server indexer running in the isolate, and a sandboxed
+// plugin pushing through the API. It has up to three scopes:
+//
+//   semanticTexts  text chunks → embeddings + keyword search
+//   tags           filterable key/values, merged into the item's queryable tags
+//   metadata       arbitrary structured data (an audiobook's chapter index, …)
+//
+// Both halves of the contract live here because they must not drift: `normalize`
+// decides what the shape IS, and `clamp` decides how large it may be. They used to sit
+// in different modules — normalize in the indexing coordinator, clamp inside the plugin
+// isolate runtime — which meant the size limits only applied to one of the three
+// producers. The caps are a property of the contribution, not of whoever made it.
+
+/** Output caps applied to every contribution, whatever produced it. */
+export const DEFAULT_CAPS = {
+  maxSemanticTexts: 500, // number of chunks
+  maxTextChars: 100_000, // per chunk
+  maxSemanticChars: 2_000_000, // total across chunks
+  maxTags: 100, // number of tag entries
+  maxTagKeyChars: 128,
+  maxTagValueChars: 2_048,
+  maxMetadataBytes: 256 * 1024, // JSON-serialized metadata
+};
+
+/**
+ * Normalize to the three canonical scopes, accepting the legacy `{ documents, facet }`
+ * shape (documents→semanticTexts, facet→metadata).
+ */
+export function normalizeContribution(c = {}) {
+  return {
+    semanticTexts: c.semanticTexts || c.documents || [],
+    tags: c.tags || null,
+    metadata: c.metadata || c.facet || null,
+  };
+}
+
+/**
+ * Clamp a contribution to `caps`, dropping (not erroring on) whatever exceeds them.
+ *
+ * Truncating rather than rejecting is deliberate: the input is a *derived* artifact of
+ * a file someone uploaded, so an oversized contribution usually means an unusual file
+ * rather than an attack, and losing the tail of an index is much better than losing the
+ * item's indexing entirely. What it must never do is let the size of one item's
+ * contribution be chosen by whoever wrote the file.
+ */
+export function clampContribution(raw, caps = DEFAULT_CAPS) {
+  const c = { ...DEFAULT_CAPS, ...caps };
+  const out = {};
+  const src = raw && typeof raw === 'object' ? raw : {};
+
+  const texts = src.semanticTexts || src.documents;
+  if (Array.isArray(texts)) {
+    const docs = [];
+    let budget = c.maxSemanticChars;
+    for (const d of texts) {
+      if (docs.length >= c.maxSemanticTexts || budget <= 0) break;
+      const text = typeof d === 'string' ? d : (d && typeof d.text === 'string' ? d.text : null);
+      if (!text) continue;
+      const clipped = text.length > c.maxTextChars ? text.slice(0, c.maxTextChars) : text;
+      const room = Math.min(clipped.length, budget);
+      const finalText = room < clipped.length ? clipped.slice(0, room) : clipped;
+      budget -= finalText.length;
+      const doc = typeof d === 'string' ? { text: finalText } : { ...d, text: finalText };
+      if (doc.fields && typeof doc.fields !== 'object') delete doc.fields;
+      docs.push(doc);
+    }
+    if (docs.length) out.semanticTexts = docs;
+  }
+
+  const tags = src.tags;
+  if (tags && typeof tags === 'object' && !Array.isArray(tags)) {
+    const clean = {};
+    let n = 0;
+    for (const [k, v] of Object.entries(tags)) {
+      if (n >= c.maxTags) break;
+      if (typeof k !== 'string' || k.length > c.maxTagKeyChars) continue;
+      const cv = clampTagValue(v, c.maxTagValueChars);
+      if (cv === undefined) continue;
+      clean[k] = cv;
+      n++;
+    }
+    if (n) out.tags = clean;
+  }
+
+  const metadata = src.metadata || src.facet;
+  if (metadata && typeof metadata === 'object') {
+    try {
+      const json = JSON.stringify(metadata);
+      // BYTES, as the cap's name says. `json.length` counts UTF-16 code units, so
+      // non-ASCII metadata could run to roughly three times the advertised limit.
+      if (json && new TextEncoder().encode(json).length <= c.maxMetadataBytes) {
+        out.metadata = JSON.parse(json);
+      }
+    } catch { /* non-serializable metadata is dropped */ }
+  }
+
+  return out;
+}
+
+/**
+ * A tag value must be a filterable scalar. Objects and arrays are dropped — they can't
+ * be compared by a `#key:>=value` filter, so storing them would only bloat the row.
+ * `null` survives: it is the documented "this tag is removed" sentinel that
+ * mergeContributionTags reads, so clamping it away would make a contribution unable to
+ * retract a tag it previously set.
+ */
+export function clampTagValue(v, maxChars) {
+  if (typeof v === 'string') return v.length > maxChars ? v.slice(0, maxChars) : v;
+  if (typeof v === 'number') return Number.isFinite(v) ? v : undefined; // NaN/∞ don't compare
+  if (typeof v === 'boolean' || v === null) return v;
+  return undefined;
+}

package/packages/core/src/indexers/registry.js

@@ -0,0 +1,162 @@
+// IndexerRegistry — server-side indexers that extract structured content from
+// files. An indexer declares which nodes it handles and, for each, produces a
+// *contribution* with up to three scopes:
+//   - semanticTexts: text chunks that become embeddings + keyword entries (search)
+//   - tags:          filterable key/values (merged into the node's queryable tags)
+//   - metadata:      arbitrary structured data (e.g. an audiobook's chapter index)
+// Everything is namespaced under the indexer's `id`, so indexers never clobber one
+// another and an indexer's whole contribution can be removed independently.
+//
+// This complements *plugin* indexers, which run in the browser sandbox and push
+// the same contribution shape through the API under their own namespace.
+//
+// Two built-ins ship here: a text/markdown extractor (the reference indexer) and a
+// links extractor, which is what makes a drive with no folders navigable.
+
+import { extname } from '../util.js';
+import { extractTroveLinks } from '../links.js';
+import { LINKS_CONTRIBUTOR, LINKS_KEY } from '../metadata/interface.js';
+
+/**
+ * @typedef {object} Indexer
+ * @property {string} id                 unique namespace
+ * @property {string} [displayName]
+ * @property {(node: object) => boolean} match
+ * @property {(node: object, ctx: IndexContext) => Promise<Contribution>} index
+ *
+ * @typedef {object} Contribution
+ * @property {Doc[]} [semanticTexts]  text chunks → embeddings + keyword search
+ * @property {object} [tags]          filterable key/values, e.g. { language: 'en' }
+ * @property {object} [metadata]      arbitrary structured data, e.g. { chapters: [...] }
+ *
+ * @typedef {object} Doc
+ * @property {string} [id]   stable per (node, chunk); defaults to `${node.id}:${i}`
+ * @property {string} text
+ * @property {object} [fields]  e.g. { title, page, chapter }
+ *
+ * @typedef {object} IndexContext
+ * @property {() => Promise<Uint8Array>} readBytes
+ * @property {() => Promise<string>} readText
+ * @property {number} maxBytes
+ */
+
+export class IndexerRegistry {
+  /** @param {{builtins?: boolean}} [opts] register the reference text indexer (default on) */
+  constructor({ builtins = true } = {}) {
+    this.indexers = new Map();
+    if (builtins) {
+      this.register(textIndexer);
+      this.register(linksIndexer);
+    }
+  }
+  register(indexer) {
+    if (!indexer?.id || typeof indexer.index !== 'function') {
+      throw new Error('Indexer requires an id and an index() function');
+    }
+    this.indexers.set(indexer.id, indexer);
+    // Only unregister if THIS registration is still the live one. Indexer ids are
+    // contribution URIs, which are account-independent, so a second account installing
+    // the same package replaces the first's entry — and the first's stale closure then
+    // deleted the second's, taking that account's indexer out of the drive-wide registry
+    // and purging its contributions.
+    return () => { if (this.indexers.get(indexer.id) === indexer) this.indexers.delete(indexer.id); };
+  }
+  unregister(id) {
+    this.indexers.delete(id);
+  }
+  list() {
+    return [...this.indexers.values()].map((i) => ({ id: i.id, displayName: i.displayName || i.id }));
+  }
+  matching(node) {
+    return [...this.indexers.values()].filter((i) => {
+      try {
+        return i.match(node);
+      } catch {
+        return false;
+      }
+    });
+  }
+}
+
+// --- Reference indexer: plain text / markdown / code ------------------------
+
+const TEXT_EXTS = new Set([
+  '.txt', '.md', '.markdown', '.rst', '.log', '.csv', '.tsv',
+  '.json', '.yaml', '.yml', '.toml', '.ini', '.env',
+  '.js', '.mjs', '.ts', '.jsx', '.tsx', '.py', '.rb', '.go', '.rs', '.java',
+  '.c', '.h', '.cpp', '.css', '.html', '.xml', '.sh',
+]);
+
+function isTexty(node) {
+  if (node.contentType?.startsWith('text/')) return true;
+  return TEXT_EXTS.has(extname(node.name));
+}
+
+export const textIndexer = {
+  id: 'core.text',
+  displayName: 'Text & code',
+  match: isTexty,
+  async index(node, ctx) {
+    const text = await ctx.readText();
+    if (!text.trim()) return { semanticTexts: [] };
+    const chunks = chunkText(text, 1200, 200);
+    // Fields describe the CHUNK (which chunk it is), not the node. The node's name
+    // deliberately isn't copied in: field values are indexed, so a name duplicated
+    // across every chunk goes stale the moment the item is renamed — leaving the file
+    // findable under a name it no longer has. Name search belongs to the `core.name`
+    // index, which is a single document and is refreshed on rename.
+    const semanticTexts = chunks.map((chunk, i) => ({
+      id: `${node.id}:${i}`,
+      text: chunk,
+      fields: { chunk: i },
+    }));
+    return {
+      semanticTexts,
+      metadata: { chars: text.length, chunks: chunks.length, excerpt: text.slice(0, 280) },
+    };
+  },
+};
+
+// --- Links indexer: what this item points at --------------------------------
+//
+// With no folder hierarchy, an item's place in the drive is defined by what links to
+// it. This indexer records the outbound `trove:` URIs found in an item's text, which
+// is what MetadataStore.findLinksTo reads to answer the inverse question — "what
+// gathers this up?" — for the backlinks panel.
+//
+// It stores the links as `metadata`, not `semanticTexts`: a URI is an edge, not prose,
+// and embedding it would only pollute semantic results. `links` is also surfaced as a
+// tag count so `#links > 0` finds the documents that act as indexes.
+
+export const linksIndexer = {
+  id: LINKS_CONTRIBUTOR,
+  displayName: 'Links',
+  match: isTexty,
+  async index(node, ctx) {
+    const text = await ctx.readText();
+    const links = extractTroveLinks(text).map((l) => l.uri);
+    return {
+      metadata: { [LINKS_KEY]: links },
+      tags: { links: links.length },
+    };
+  },
+};
+
+/** Split into overlapping windows on sentence/line boundaries where possible. */
+export function chunkText(text, size, overlap) {
+  const out = [];
+  let i = 0;
+  while (i < text.length) {
+    let end = Math.min(i + size, text.length);
+    if (end < text.length) {
+      // Prefer to break at a paragraph/sentence boundary within the last 25%.
+      const window = text.slice(i, end);
+      const brk = Math.max(window.lastIndexOf('\n\n'), window.lastIndexOf('. '), window.lastIndexOf('\n'));
+      if (brk > size * 0.75) end = i + brk + 1;
+    }
+    out.push(text.slice(i, end).trim());
+    if (end >= text.length) break;
+    i = end - overlap;
+  }
+  return out.filter(Boolean);
+}