@wei840222/qmd 2026.8.28 → 2026.9.25

package/dist/metadata.js

@@ -0,0 +1,215 @@
+/**
+ * QMD Metadata - Public metadata types and frontmatter extraction.
+ *
+ * Documents opt into metadata through a namespaced Markdown frontmatter block:
+ *
+ *   ---
+ *   qmd:
+ *     metadata:
+ *       topics:
+ *         - typescript
+ *         - programming
+ *       status: published
+ *   ---
+ *
+ * Extraction is source-agnostic at the persistence boundary: this module
+ * produces a canonical `MetadataExtractionResult`, and future non-frontmatter
+ * sources can produce the same shape without touching storage or filtering.
+ *
+ * The raw document is never modified — frontmatter stays part of the stored,
+ * indexed, chunked, and embedded content.
+ */
+import YAML from "yaml";
+// =============================================================================
+// Limits
+// =============================================================================
+/**
+ * Bump when extraction or normalization semantics change so existing rows are
+ * re-extracted on the next `qmd update`.
+ */
+export const METADATA_EXTRACTION_VERSION = 1;
+/** Defensive limits for metadata from untrusted repositories. */
+export const METADATA_LIMITS = {
+    maxFrontmatterBytes: 64 * 1024,
+    maxKeys: 64,
+    maxKeyBytes: 128,
+    maxStringLength: 1024,
+    maxArrayLength: 128,
+    maxYamlAliasCount: 100,
+    maxErrorLength: 200,
+};
+/** File extensions parsed for frontmatter metadata. */
+const FRONTMATTER_FILE_EXTENSIONS = new Set([".md", ".markdown", ".mdx"]);
+// =============================================================================
+// Extraction
+// =============================================================================
+/**
+ * Extract `qmd.metadata` from a document's leading YAML frontmatter.
+ *
+ * Never throws. A document without frontmatter, without a `qmd` namespace, or
+ * with a non-frontmatter extension yields empty metadata with no error.
+ * Invalid frontmatter or invalid metadata yields empty metadata plus a bounded
+ * extraction error.
+ */
+export function extractDocumentMetadata(content, path) {
+    const success = (metadata) => ({ metadata, extractionVersion: METADATA_EXTRACTION_VERSION });
+    const failure = (message) => ({
+        metadata: {},
+        error: truncateErrorMessage(message),
+        extractionVersion: METADATA_EXTRACTION_VERSION,
+    });
+    if (!hasFrontmatterFileExtension(path))
+        return success({});
+    const frontmatterYaml = getFrontmatterYaml(content);
+    if (frontmatterYaml === null)
+        return success({});
+    if (Buffer.byteLength(frontmatterYaml, "utf-8") > METADATA_LIMITS.maxFrontmatterBytes) {
+        return failure(`frontmatter exceeds ${METADATA_LIMITS.maxFrontmatterBytes} bytes`);
+    }
+    let frontmatter;
+    try {
+        frontmatter = YAML.parse(frontmatterYaml, { maxAliasCount: METADATA_LIMITS.maxYamlAliasCount });
+    }
+    catch (err) {
+        return failure(`invalid frontmatter YAML: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    if (!isPlainObject(frontmatter))
+        return success({});
+    const qmdNamespace = frontmatter["qmd"];
+    if (qmdNamespace === undefined)
+        return success({});
+    if (!isPlainObject(qmdNamespace)) {
+        return failure("frontmatter 'qmd' must be a mapping");
+    }
+    const rawMetadata = qmdNamespace["metadata"];
+    if (rawMetadata === undefined)
+        return success({});
+    if (!isPlainObject(rawMetadata)) {
+        return failure("frontmatter 'qmd.metadata' must be a mapping");
+    }
+    try {
+        return success(normalizeMetadata(rawMetadata));
+    }
+    catch (err) {
+        return failure(err instanceof Error ? err.message : String(err));
+    }
+}
+function hasFrontmatterFileExtension(path) {
+    const dotIndex = path.lastIndexOf(".");
+    if (dotIndex < 0)
+        return false;
+    return FRONTMATTER_FILE_EXTENSIONS.has(path.slice(dotIndex).toLowerCase());
+}
+/**
+ * Slice the YAML between a leading `---` line and a closing `---` or `...`
+ * line. Tolerates a UTF-8 BOM and CRLF line endings. Returns null when the
+ * document has no complete leading frontmatter block.
+ */
+function getFrontmatterYaml(content) {
+    const body = content.charCodeAt(0) === 0xfeff ? content.slice(1) : content;
+    const openMatch = body.match(/^---[ \t]*\r?\n/);
+    if (!openMatch)
+        return null;
+    const yamlStart = openMatch[0].length;
+    const closePattern = /^(?:---|\.\.\.)[ \t]*(?:\r?\n|$)/m;
+    const closeMatch = body.slice(yamlStart).match(closePattern);
+    if (!closeMatch || closeMatch.index === undefined)
+        return null;
+    return body.slice(yamlStart, yamlStart + closeMatch.index);
+}
+// =============================================================================
+// Normalization
+// =============================================================================
+/**
+ * Normalize raw `qmd.metadata` into canonical `DocumentMetadata`.
+ * Throws on any unsupported key or value — extraction is all-or-nothing per
+ * document so stale partial metadata can never persist.
+ */
+function normalizeMetadata(rawMetadata) {
+    const keys = Object.keys(rawMetadata);
+    if (keys.length > METADATA_LIMITS.maxKeys) {
+        throw new Error(`metadata has ${keys.length} keys (max ${METADATA_LIMITS.maxKeys})`);
+    }
+    const entries = [];
+    for (const key of keys) {
+        validateMetadataKey(key);
+        entries.push([key, normalizeMetadataValue(key, rawMetadata[key])]);
+    }
+    // Object.fromEntries defines "__proto__" as ordinary data instead of
+    // invoking Object.prototype's legacy setter. Metadata keys are unrestricted
+    // user data, so prototype-shaped names must round-trip like every other key.
+    return Object.fromEntries(entries);
+}
+function validateMetadataKey(key) {
+    if (key.length === 0) {
+        throw new Error("metadata keys must be non-empty strings");
+    }
+    if (Buffer.byteLength(key, "utf-8") > METADATA_LIMITS.maxKeyBytes) {
+        throw new Error(`metadata key exceeds ${METADATA_LIMITS.maxKeyBytes} bytes`);
+    }
+    // eslint-disable-next-line no-control-regex
+    if (/[\u0000-\u001f\u007f]/.test(key)) {
+        throw new Error("metadata keys must not contain control characters");
+    }
+}
+function normalizeMetadataValue(key, rawValue) {
+    if (Array.isArray(rawValue)) {
+        return normalizeMetadataArray(key, rawValue);
+    }
+    return normalizeMetadataScalar(key, rawValue);
+}
+function normalizeMetadataScalar(key, rawValue) {
+    if (typeof rawValue === "string") {
+        if (rawValue.length > METADATA_LIMITS.maxStringLength) {
+            throw new Error(`metadata key "${key}": string exceeds ${METADATA_LIMITS.maxStringLength} characters`);
+        }
+        return rawValue;
+    }
+    if (typeof rawValue === "number") {
+        if (!Number.isFinite(rawValue)) {
+            throw new Error(`metadata key "${key}": numbers must be finite`);
+        }
+        return rawValue;
+    }
+    if (typeof rawValue === "boolean")
+        return rawValue;
+    if (rawValue === null) {
+        throw new Error(`metadata key "${key}": null is not supported — omit the key instead`);
+    }
+    throw new Error(`metadata key "${key}": unsupported value type — use strings, numbers, booleans, or flat arrays of one of those`);
+}
+function normalizeMetadataArray(key, rawValues) {
+    if (rawValues.length === 0) {
+        throw new Error(`metadata key "${key}": empty arrays are not supported — omit the key instead`);
+    }
+    if (rawValues.length > METADATA_LIMITS.maxArrayLength) {
+        throw new Error(`metadata key "${key}": array exceeds ${METADATA_LIMITS.maxArrayLength} values`);
+    }
+    const scalars = rawValues.map(rawValue => {
+        if (Array.isArray(rawValue)) {
+            throw new Error(`metadata key "${key}": nested arrays are not supported`);
+        }
+        return normalizeMetadataScalar(key, rawValue);
+    });
+    // Narrow each homogeneous case explicitly so the public array union remains
+    // precise without discarding type evidence through chained assertions.
+    if (scalars.every((scalar) => typeof scalar === "string")) {
+        return Array.from(new Set(scalars));
+    }
+    if (scalars.every((scalar) => typeof scalar === "number")) {
+        return Array.from(new Set(scalars));
+    }
+    if (scalars.every((scalar) => typeof scalar === "boolean")) {
+        return Array.from(new Set(scalars));
+    }
+    throw new Error(`metadata key "${key}": mixed-type arrays are not supported`);
+}
+function isPlainObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function truncateErrorMessage(message) {
+    const singleLine = message.replace(/\s+/g, " ").trim();
+    if (singleLine.length <= METADATA_LIMITS.maxErrorLength)
+        return singleLine;
+    return singleLine.slice(0, METADATA_LIMITS.maxErrorLength - 3) + "...";
+}

package/dist/search/zh-dict.txt

@@ -179849,6 +179849,7 @@ c++ 3 nz
 實際性 4 N
 實際數 3 N
 實際節 1 N
+實際耗時 1000000 nz
 實際面 6 N
 實領 4 N
 實馬 1 N
@@ -403350,6 +403351,7 @@ c++ 3 nz
 真實度 3 N
 真實性 86 N
 真實感 20 N
+真實時間 1000000 nz
 真實模式 1000000 nz
 真實版 16 N
 真實面 7 N
@@ -437457,6 +437459,7 @@ c++ 3 nz
 總編輯 112 N
 總署 191 N
 總而言之 20 ADV
+總耗時 1000000 nz
 總膽管 3 N
 總荷 1 N
 總菌 3 N

package/dist/store.d.ts

@@ -18,6 +18,8 @@ import type { LLM } from "./llm.js";
 import { LlamaCpp, formatQueryForEmbedding, formatDocForEmbedding, type ILLMSession } from "./llm.js";
 import type { NamedCollection, Collection, CollectionConfig } from "./collections.js";
 import type { IndexDiagnostics } from "./diagnostics.js";
+import { type DocumentMetadata } from "./metadata.js";
+import { type MetadataFilter } from "./metadata-filter.js";
 export declare const DEFAULT_EMBED_MODEL = "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf";
 export declare const DEFAULT_RERANK_MODEL = "hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf";
 export declare const DEFAULT_QUERY_MODEL = "hf:tobil/qmd-query-expansion-1.7B-gguf/qmd-query-expansion-1.7B-q4_k_m.gguf";
@@ -310,9 +312,9 @@ export type Store = {
     isVirtualPath: typeof isVirtualPath;
     resolveVirtualPath: (virtualPath: string) => string | null;
     toVirtualPath: (absolutePath: string) => string | null;
-    searchCharFTS: (query: string, limit?: number, collectionFilter?: CollectionFilter) => SearchResult[];
-    searchFTS: (query: string, limit?: number, collectionFilter?: CollectionFilter) => SearchResult[];
-    searchVec: (query: string, model: string, limit?: number, collectionFilter?: CollectionFilter, session?: ILLMSession, precomputedEmbedding?: number[]) => Promise<SearchResult[]>;
+    searchCharFTS: (query: string, limit?: number, collectionFilter?: CollectionFilter, filter?: MetadataFilter) => SearchResult[];
+    searchFTS: (query: string, limit?: number, collectionFilter?: CollectionFilter, filter?: MetadataFilter) => SearchResult[];
+    searchVec: (query: string, model: string, limit?: number, collectionFilter?: CollectionFilter, session?: ILLMSession, precomputedEmbedding?: number[], filter?: MetadataFilter) => Promise<SearchResult[]>;
     expandQuery: (query: string, model?: string, expansionContext?: string, options?: QueryExpansionOptions) => Promise<ExpandedQuery[]>;
     /** Drop the cached expansion for a query so the next call regenerates. */
     invalidateExpansionCache: (query: string, expansionContext?: string, options?: QueryExpansionOptions) => void;
@@ -347,7 +349,7 @@ export type Store = {
         hash: string;
     } | null;
     insertContent: (hash: string, content: string, createdAt: string) => void;
-    insertDocument: (collectionName: string, path: string, title: string, hash: string, createdAt: string, modifiedAt: string) => void;
+    insertDocument: (collectionName: string, path: string, title: string, hash: string, createdAt: string, modifiedAt: string) => number;
     findActiveDocument: (collectionName: string, path: string) => {
         id: number;
         hash: string;
@@ -367,6 +369,7 @@ export type Store = {
         body: string;
         path: string;
     }[];
+    insertEmbedding: (hash: string, seq: number, pos: number, embedding: Float32Array, model: string, embeddedAt: string, totalChunks?: number, fingerprint?: string, lease?: EmbeddingBuildLease) => void;
 };
 export type ReindexProgress = {
     file: string;
@@ -385,6 +388,8 @@ export type ReindexResult = {
     orphanedCleaned: number;
     skipped: number;
     skippedFiles: ReindexSkippedFile[];
+    /** Documents whose qmd.metadata frontmatter failed extraction this pass. */
+    metadataErrors: number;
 };
 /**
  * Re-index a single collection by scanning the filesystem and updating the database.
@@ -501,6 +506,7 @@ export declare function handelize(path: string): string;
  * Search result extends DocumentResult with score and source info
  */
 export type SearchResult = DocumentResult & {
+    metadata: DocumentMetadata;
     score: number;
     source: "fts" | "vec";
     chunkPos?: number;
@@ -612,6 +618,8 @@ export type IndexStatus = {
     totalDocuments: number;
     needsEmbedding: number;
     hasVectorIndex: boolean;
+    /** Active documents without current, error-free metadata extraction. */
+    pendingMetadata: number;
     collections: CollectionInfo[];
     /** Additive diagnostics populated by high-level composition roots. */
     diagnostics?: IndexDiagnostics;
@@ -707,10 +715,11 @@ export declare function extractTitle(content: string, filename: string): string;
 export declare function insertContent(db: Database, hash: string, content: string, createdAt: string): void;
 /**
  * Insert a new document into the documents table.
+ * Returns the document's id so callers can attach document-scoped state.
  */
-export declare function insertDocument(db: Database, collectionName: string, path: string, title: string, hash: string, createdAt: string, modifiedAt: string): void;
+export declare function insertDocument(db: Database, collectionName: string, path: string, title: string, hash: string, createdAt: string, modifiedAt: string): number;
 /** Insert immutable content and its document row in the same lexical transaction. */
-export declare function insertDocumentWithContent(db: Database, hash: string, content: string, contentCreatedAt: string, collectionName: string, path: string, title: string, documentCreatedAt: string, modifiedAt: string): void;
+export declare function insertDocumentWithContent(db: Database, hash: string, content: string, contentCreatedAt: string, collectionName: string, path: string, title: string, documentCreatedAt: string, modifiedAt: string): number;
 /**
  * Find an active document by collection name and path.
  */
@@ -778,13 +787,6 @@ export declare function chunkDocumentAsync(content: string, maxChars?: number, o
     text: string;
     pos: number;
 }[]>;
-/**
- * Chunk a document by actual token count using the LLM tokenizer.
- * More accurate than character-based chunking but requires async.
- *
- * When filepath and chunkStrategy are provided, uses AST-aware break points
- * for supported code files.
- */
 export declare function chunkDocumentByTokens(content: string, maxTokens?: number, overlapTokens?: number, windowTokens?: number, filepath?: string, chunkStrategy?: ChunkStrategy, signal?: AbortSignal): Promise<{
     text: string;
     pos: number;
@@ -929,9 +931,9 @@ export declare const CJK_LEXICAL_CANDIDATE_DEPTH = 60;
 export declare const CJK_LEXICAL_RRF_WEIGHTS: Readonly<Record<CjkLexicalChannel, number>>;
 export type CollectionFilter = string | readonly string[];
 export type CollectionScope = string | readonly string[] | undefined;
-export declare function searchCharFTS(db: Database, query: string, limit?: number, collectionFilter?: CollectionFilter): SearchResult[];
-export declare function searchFTS(db: Database, query: string, limit?: number, collectionFilter?: CollectionFilter): SearchResult[];
-export declare function searchVec(db: Database, query: string, model: string, limit?: number, collectionFilter?: CollectionFilter, session?: ILLMSession, precomputedEmbedding?: number[], provider?: EmbeddingProvider, authorizeRemoteRequest?: Store["authorizeRemoteRequest"], llmOverride?: LLM): Promise<SearchResult[]>;
+export declare function searchCharFTS(db: Database, query: string, limit?: number, collectionFilter?: CollectionFilter, filter?: MetadataFilter): SearchResult[];
+export declare function searchFTS(db: Database, query: string, limit?: number, collectionFilter?: CollectionScope, filter?: MetadataFilter): SearchResult[];
+export declare function searchVec(db: Database, query: string, model: string, limit?: number, collectionFilter?: CollectionFilter, session?: ILLMSession, precomputedEmbedding?: number[], providerOrLlm?: EmbeddingProvider | LLM, authorizeRemoteRequestOrFilter?: Store["authorizeRemoteRequest"] | MetadataFilter, llmOverride?: LLM, metadataFilter?: MetadataFilter): Promise<SearchResult[]>;
 /**
  * Get all unique content hashes that need embeddings (from active documents).
  * Returns hash, document body, and a sample path for display purposes.
@@ -1104,6 +1106,7 @@ export type ExpansionErrorEvent = {
 export interface HybridQueryOptions {
     collection?: string | readonly string[];
     collections?: readonly string[];
+    filter?: MetadataFilter;
     limit?: number;
     minScore?: number;
     candidateLimit?: number;
@@ -1128,6 +1131,7 @@ export interface HybridQueryResult {
     score: number;
     context: string | null;
     docid: string;
+    metadata: DocumentMetadata;
     explain?: HybridQueryExplain;
 }
 export type RankedListMeta = {
@@ -1163,6 +1167,7 @@ export declare function getHybridRrfWeights(rankedListMeta: RankedListMeta[]): n
 export declare function hybridQuery(store: Store, query: string, options?: HybridQueryOptions): Promise<HybridQueryResult[]>;
 export interface VectorSearchOptions {
     collection?: CollectionFilter;
+    filter?: MetadataFilter;
     limit?: number;
     minScore?: number;
     /** Additional context used only while generating query expansions. */
@@ -1179,6 +1184,7 @@ export interface VectorSearchResult {
     score: number;
     context: string | null;
     docid: string;
+    metadata: DocumentMetadata;
 }
 /**
  * Vector-only semantic search with query expansion.
@@ -1196,6 +1202,7 @@ export declare function vectorSearchQuery(store: Store, query: string, options?:
  */
 export interface StructuredSearchOptions {
     collections?: string[];
+    filter?: MetadataFilter;
     limit?: number;
     minScore?: number;
     candidateLimit?: number;