greptor 0.2.1

package/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2026 Sergii Vashchyshchuk
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

package/README.md

@@ -0,0 +1,365 @@
+# Greptor
+
+> **Grep + Raptor**: Transform messy, unstructured text into clean, grep-friendly data for agentic search workflows.
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+[![npm version](https://badge.fury.io/js/greptor.svg)](https://www.npmjs.com/package/greptor)
+
+Claude Code has proven that agentic search (`ripgrep` + filesystem traversal + iterative investigation) is powerful enough for complex code navigation tasks. But what about **textual data** like documents, transcripts, posts, articles, notes, and reports?
+
+Greptor is a library that helps you with this. It **ingests and indexes unstructured text** into a format that agents can easily search using simple tools like `ripgrep`.
+
+## Why Agentic Search (and Why Not Classic RAG)?
+
+RAG worked around small context windows by chunking documents and retrieving "relevant" fragments. That approach has recurring pain points:
+
+- **Chunking breaks structure**: Tables, section hierarchies, and cross-references get lost.
+- **Embeddings are fuzzy**: They struggle with exact terms, numbers, and identifiers.
+- **Complexity overhead**: Hybrid search + rerankers add latency, cost, and moving parts.
+- **Error cascade**: If retrieval misses the right chunk, the answer can't be correct.
+
+Agentic search flips the approach: with larger context windows and better tool use, agents can **search, open files, follow references, and refine queries** — more like a human analyst.
+
+Greptor's job is to clean, chunk, and add structure to your documents, making them **easily searchable with text tools** like `ripgrep`. No complex indices, no retrievers, no vector databases. Just **minimal initial processing + maximal grep-ability**.
+
+## How It Works
+
+### Step 1: Install
+
+```bash
+npm install greptor
+# or
+bun add greptor
+```
+
+### Step 2: Initialize
+
+Create a Greptor instance with your base directory, topic, and LLM model.
+
+```typescript
+import { createGreptor } from 'greptor';
+
+// Create Greptor instance
+const greptor = await createGreptor({
+  baseDir: './projects/investing',
+  topic: 'Investing, stock market, financial, and macroeconomics',
+  llmModel: 'openai:gpt-5-mini'
+});
+```
+
+- **`baseDir`**: Home directory for your project where all data will be stored.
+- **`topic`**: Helps Greptor understand your data better and generate a relevant metadata schema.
+- **`llmModel`**: OpenAI-compatible model for chunking and metadata extraction. You must provide an API key via environment variables.
+
+### Step 3: Start Feeding Documents
+
+```typescript
+await greptor.eat({
+  id: 'QwwVJfvfqN8',
+  source: 'youtube',
+  publisher: '@JosephCarlsonShow',
+  format: 'text',
+  label: 'Top Five AI Stocks I\'m Buying Now',
+  content: '{fetch and populate video transcript here}',
+  creationDate: new Date('2025-11-15'),
+  metadata: {
+    // Optional custom metadata specific to the source or document
+    channelTitle: 'Joseph Carlson',
+    channelSubscribers: 496000
+  },
+});
+
+await greptor.eat({
+  id: 'tesla_reports_418227_deliveries_for_the_fourth',
+  source: 'reddit',
+  publisher: 'investing',  // For Reddit, publisher is the subreddit name
+  format: 'text',
+  label: 'Tesla reports 418,227 deliveries for the fourth quarter, down 16%',
+  content: '{fetch and populate Reddit post with comments here}',
+  creationDate: new Date('2025-12-03'),
+  metadata: {
+    // Optional custom metadata
+    upvotes: 1400
+  },
+});
+```
+
+### Step 4: Wait for Background Processing
+
+Greptor will write your input to a raw Markdown file immediately, then run background enrichment (LLM cleaning + chunking + metadata extraction) and write a processed Markdown file. You can grep the raw files right away, and the processed files will appear shortly after.
+
+### Step 5: Generate a Claude Code Skill
+
+```typescript
+await greptor.createSkill(['youtube', 'reddit']);
+```
+
+This generates a Claude Code skill that instructs agents on how to search your indexed content effectively.
+
+The skill is customized for the sources you provide and includes search tips based on the metadata schema. You can always customize it manually further for better results.
+
+### Step 6: Run the Agent
+
+By this point, you should have the following structure in your `baseDir`:
+
+```
+./projects/investing/
+  .claude/
+    skills/
+      search-youtube-reddit/
+        SKILL.md
+  content/
+    raw/
+      youtube/
+        JosephCarlsonShow/
+          2025-12/
+            2025-12-01-Top-Five-AI-Stocks-Im-Buying-Now.md
+      reddit/
+        investing/
+          2025-12/
+            2025-12-03-Tesla-reports-418227-deliveries-for-the-fourth-quarter-down-16.md
+    processed/
+      youtube/
+        JosephCarlsonShow/
+          2025-12/
+            2025-12-01-Top-Five-AI-Stocks-Im-Buying-Now.md
+      reddit/
+        investing/
+          2025-12/
+            2025-12-03-Tesla-reports-418227-deliveries-for-the-fourth-quarter-down-16.md
+```
+
+Now run Claude Code (or any other agent) in this folder and ask questions about your data or perform research tasks!
+
+**Note**: For other agents, you may need to adapt the skill accordingly.
+
+**For better results**:
+1. Connect MCP servers like Yahoo Finance or other relevant financial/stock market MCP servers for up-to-date information.
+2. Add personal financial information, such as your portfolio holdings, watchlists, and risk profile.
+3. Create custom skills, slash commands, or subagents for researching specific tickers, sectors, topics, or managing your portfolio.
+
+Now you have a personal investment research assistant with access to your portfolio, sentiment data (YouTube, Reddit), news, and market data! You don't have to manually watch dozens of YouTube channels or spend hours scrolling Reddit and other sources.
+
+## Under the Hood
+
+### 1) Raw Write (Immediate)
+
+`eat()` writes the input to a raw Markdown file with YAML frontmatter. You can grep it right away.
+
+### 2) Background Processing (Asynchronous)
+
+Workers pick up new documents and run a one-time pipeline:
+
+1. **LLM cleaning**: Remove timestamps, ads, disclaimers, boilerplate, and irrelevant content.
+2. **LLM chunking**: Transform a blob into semantic section chunks.
+3. **LLM metadata extraction**: Extract metadata relevant to your topic/domain and enrich each chunk with denormalized metadata.
+
+Here's an example of a processed file:
+
+```markdown
+---
+title: "NVIDIA Q4 2024 Earnings: AI Boom Continues"
+source: "youtube"
+publisher: "Wall Street Millennial"
+date: 2025-11-15
+ticker: "NVDA"
+videoId: "dQw4w9WgXcQ"
+url: "https://youtube.com/watch?v=dQw4w9WgXcQ"
+chunks:
+  - id: c01
+    title: "Revenue Growth Analysis"
+    topics: [earnings, revenue, data-center]
+    sentiment: positive
+    tickers: [NVDA]
+    price_mentioned_usd: 850.50
+    revenue_mentioned_billions: 35.1
+  - id: c02
+    title: "AI Chip Demand Outlook"
+    topics: [ai, competition, market-share]
+    sentiment: bullish
+    tickers: [NVDA, AMD, INTC]
+    timeframe: next-quarter
+---
+
+CHUNK c01: "Revenue Growth Analysis"
+NVIDIA reported Q4 revenue of $35.1 billion, beating estimates...
+
+CHUNK c02: "AI Chip Demand Outlook"
+The demand for AI accelerators continues to outpace supply...
+```
+
+### 3) Navigate with grep/glob
+
+Your "index" is the YAML frontmatter combined with the file layout. Agents can search it deterministically.
+
+**Search examples**:
+
+```bash
+# Find all bullish sentiment for TSLA stock
+rg -l "ticker:.*TSLA" content/processed | xargs rg "sentiment:.*bullish"
+
+# Count documents per ticker
+rg "ticker:" content/processed -o | sort | uniq -c | sort -rn | head -20
+
+# What companies does a specific YouTuber discuss?
+rg "company:" content/processed/youtube/JosephCarlsonShow -o | sort | uniq -c | sort -rn
+
+# Find all AI-related narratives with strong buy recommendations
+rg -l "narrative:.*ai_boom" content/processed | xargs rg "recommendation:.*strong_buy"
+
+# Technology sector stocks with bullish sentiment in December 2025
+rg -l "sector:.*technology" content/processed --glob "**/2025-12/*.md" | xargs rg "sentiment:.*bullish"
+
+# Find dividend investment style discussions
+rg "investment_style:.*dividend" content/processed -l | head -10
+
+# Bearish sentiment on large-cap stocks
+rg -l "market_cap:.*large_cap" content/processed | xargs rg "sentiment:.*bearish"
+
+# List all tickers mentioned with their sentiment
+rg "ticker: \[.*\]" content/processed -A 5 | rg "sentiment:"
+
+# Find EV-related discussions across all sources
+rg "narrative:.*ev_transition" content/processed
+
+# Combine multiple filters: tech stocks with strong buy in specific timeframe
+rg -l "sector:.*technology" content/processed --glob "**/2025-11/*.md" | \
+  xargs rg -l "recommendation:.*strong_buy" | \
+  xargs rg "ticker:" -o | sort | uniq -c
+```
+
+**Analysis patterns**:
+
+```bash
+# Aggregate sentiment distribution
+rg "sentiment:" content/processed -o | cut -d: -f2 | tr -d ' ' | sort | uniq -c
+
+# Most discussed sectors
+rg "sector:" content/processed -o | sort | uniq -c | sort -rn
+
+# Track narrative evolution over time
+for month in 2025-{10..12}; do
+  echo "=== $month ==="
+  rg "narrative:" content/processed --glob "**/$month-*/*.md" -o | sort | uniq -c | sort -rn | head -5
+done
+
+# Compare sentiment on specific stock across sources
+for source in youtube reddit; do
+  echo "=== $source ==="
+  rg -l "ticker:.*AAPL" content/processed/$source | xargs rg "sentiment:" -o | sort | uniq -c
+done
+```
+
+## Configuration
+
+### LLM Model Format
+
+Greptor uses the following LLM model format: `provider:model-name`
+
+The provider is an OpenAI API-compatible provider, such as `openai`, `azure`, `ollama`, `deepseek`, etc.
+
+**Examples**:
+
+```typescript
+llmModel: 'openai:gpt-5-mini'
+llmModel: 'ollama:llama3-70b'
+```
+
+**Important**: Use a model at least at the level of GPT-5-mini or better.
+**Required Environment Variables**:
+
+```bash
+# For OpenAI models
+OPENAI_API_KEY=your_key_here
+
+# For Azure OpenAI models
+AZURE_API_KEY=your_key_here
+AZURE_API_BASE_URL=https://your-azure-endpoint.com/v1
+```
+
+## Metadata Schemas
+
+If you don't provide a schema, Greptor can initialize one for your topic. However, for better results, provide a custom schema.
+
+Here's a comprehensive example for investment research:
+
+```typescript
+const greptor = await createGreptor({
+  baseDir: './projects/investing',
+  topic: 'Investing, stock market, financial, and macroeconomics',
+  llmModel: 'openai:gpt-5-mini',
+  metadataSchema: [
+    {
+      name: 'company',
+      type: 'string[]',
+      description: 'Canonical company names in snake_case (e.g. apple, tesla, microsoft)',
+    },
+    {
+      name: 'ticker',
+      type: 'string[]',
+      description: 'Canonical stock tickers, UPPERCASE only (e.g. AAPL, TSLA, MSFT, SPY)',
+    },
+    {
+      name: 'sector',
+      type: 'enum[]',
+      description: 'GICS sector classification for stocks/companies discussed',
+      enumValues: [
+        'technology', 'healthcare', 'financials', 'consumer_discretionary',
+        'consumer_staples', 'energy', 'utilities', 'industrials',
+        'materials', 'real_estate', 'communication_services',
+        'etf', 'index', 'commodity', 'bond', 'mixed'
+      ],
+    },
+    {
+      name: 'industry',
+      type: 'string[]',
+      description: 'Specific industry/sub-sector in snake_case (e.g. semiconductors, biotech, banking)',
+    },
+    {
+      name: 'market_cap',
+      type: 'enum[]',
+      description: 'Market capitalization category of the company',
+      enumValues: ['mega_cap', 'large_cap', 'mid_cap', 'small_cap', 'micro_cap'],
+    },
+    {
+      name: 'investment_style',
+      type: 'enum[]',
+      description: 'Investment approach or style discussed',
+      enumValues: [
+        'value', 'growth', 'dividend', 'momentum', 'index',
+        'passive', 'active', 'day_trading', 'swing_trading', 'long_term_hold'
+      ],
+    },
+    {
+      name: 'asset_type',
+      type: 'enum[]',
+      description: 'Type of financial instrument discussed',
+      enumValues: [
+        'stock', 'etf', 'mutual_fund', 'option', 'bond',
+        'reit', 'commodity', 'crypto', 'cash'
+      ],
+    },
+    {
+      name: 'narrative',
+      type: 'string[]',
+      description: 'Investment or market narratives in snake_case (e.g. ai_boom, ev_transition, rate_cuts)',
+    },
+    {
+      name: 'sentiment',
+      type: 'enum[]',
+      description: 'Directional stance on the stock/market',
+      enumValues: ['bullish', 'bearish', 'neutral', 'mixed', 'cautious'],
+    },
+    {
+      name: 'recommendation',
+      type: 'enum[]',
+      description: 'Analyst or influencer recommendation type',
+      enumValues: ['strong_buy', 'buy', 'hold', 'sell', 'strong_sell'],
+    },
+  ],
+});
+```
+
+## License
+
+MIT © Sergii Vashchyshchuk

package/dist/greptor.d.ts

@@ -0,0 +1,7 @@
+import type { CreateSkillResult, GreptorEatInput, GreptorEatResult, GreptorOptions } from "./types.js";
+export interface Greptor {
+    eat: (input: GreptorEatInput) => Promise<GreptorEatResult>;
+    createSkill: (sources: string[], overwrite: boolean) => Promise<CreateSkillResult>;
+}
+export declare function createGreptor(options: GreptorOptions): Promise<Greptor>;
+//# sourceMappingURL=greptor.d.ts.map

package/dist/greptor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"greptor.d.ts","sourceRoot":"","sources":["../src/greptor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,iBAAiB,EACjB,eAAe,EACf,gBAAgB,EAChB,cAAc,EACd,MAAM,YAAY,CAAC;AAcpB,MAAM,WAAW,OAAO;IACvB,GAAG,EAAE,CAAC,KAAK,EAAE,eAAe,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC3D,WAAW,EAAE,CACZ,OAAO,EAAE,MAAM,EAAE,EACjB,SAAS,EAAE,OAAO,KACd,OAAO,CAAC,iBAAiB,CAAC,CAAC;CAChC;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,CAqH7E"}

package/dist/greptor.js

@@ -0,0 +1,98 @@
+import path from "node:path";
+import YAML from "yaml";
+import { createLlmClient } from "./llm/llm-factory.js";
+import { initializeMetadataSchema } from "./metadata-schema/initialize.js";
+import { createProcessingQueue, enqueueUnprocessedDocuments, startBackgroundWorkers, } from "./processing/processor.js";
+import { generateSkill } from "./skills/skill-generator.js";
+import { createFileStorage } from "./storage/file-storage.js";
+export async function createGreptor(options) {
+    const { baseDir, logger } = options;
+    const contentPath = path.join(baseDir, "content");
+    const storage = createFileStorage(contentPath);
+    logger?.debug?.("Initializing Greptor", { baseDir, topic: options.topic });
+    const metadataSchema = await initializeMetadataSchema(storage.baseDir, options.llmModel, options.topic, options.metadataSchema, logger);
+    const queue = createProcessingQueue();
+    const queuedCount = await enqueueUnprocessedDocuments({
+        storage,
+        queue,
+        ...(logger ? { logger } : {}),
+    });
+    const llm = createLlmClient(options.llmModel);
+    const ctx = {
+        domain: options.topic,
+        metadataSchema: YAML.stringify(metadataSchema),
+        llm,
+        storage,
+        ...(logger ? { logger } : {}),
+    };
+    startBackgroundWorkers({ ctx, queue, concurrency: options.workers ?? 1 });
+    logger?.info?.("Greptor initialized", {
+        topic: options.topic,
+        queued: queuedCount,
+    });
+    async function eat(input) {
+        if (input.format !== "text") {
+            logger?.warn?.("Unsupported format", { format: input.format });
+            return {
+                success: false,
+                message: `Unsupported format: ${input.format}`,
+            };
+        }
+        const res = await storage.saveRawContent(input);
+        if (res.type === "duplicate") {
+            logger?.warn?.("Attempt to add duplicate document", {
+                ref: res.ref,
+                label: input.label,
+            });
+            return {
+                success: false,
+                message: "Document already exists.",
+            };
+        }
+        if (res.type === "error") {
+            return {
+                success: false,
+                message: res.message,
+            };
+        }
+        queue.enqueue(res.ref);
+        logger?.info?.("Document ingested", { ref: res.ref, label: input.label });
+        return {
+            success: true,
+            message: "Content added.",
+            ref: res.ref,
+        };
+    }
+    async function createSkill(sources, overwrite = false) {
+        try {
+            logger?.info?.("Generating Claude Code skill", {
+                domain: options.topic,
+            });
+            const { skillPath } = await generateSkill({
+                domain: options.topic,
+                sources,
+                baseDir: options.baseDir,
+                metadataSchema,
+                overwrite,
+            }, storage);
+            return {
+                success: true,
+                message: `Skill created at ${skillPath}`,
+                skillPath,
+            };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            logger?.error?.(`Skill generation failed:\n${errorMessage}`);
+            return {
+                success: false,
+                message: errorMessage,
+            };
+        }
+    }
+    return {
+        eat,
+        createSkill,
+    };
+}
+//# sourceMappingURL=greptor.js.map

package/dist/greptor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"greptor.js","sourceRoot":"","sources":["../src/greptor.ts"],"names":[],"mappings":"AAOA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,wBAAwB,EAAE,MAAM,iCAAiC,CAAC;AAC3E,OAAO,EACN,qBAAqB,EACrB,2BAA2B,EAC3B,sBAAsB,GACtB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAU9D,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAAuB;IAC1D,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IACpC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;IAE/C,MAAM,EAAE,KAAK,EAAE,CAAC,sBAAsB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IAE3E,MAAM,cAAc,GAAG,MAAM,wBAAwB,CACpD,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,cAAc,EACtB,MAAM,CACN,CAAC;IAEF,MAAM,KAAK,GAAG,qBAAqB,EAAE,CAAC;IACtC,MAAM,WAAW,GAAG,MAAM,2BAA2B,CAAC;QACrD,OAAO;QACP,KAAK;QACL,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC7B,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,eAAe,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC9C,MAAM,GAAG,GAAG;QACX,MAAM,EAAE,OAAO,CAAC,KAAK;QACrB,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC;QAC9C,GAAG;QACH,OAAO;QACP,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC7B,CAAC;IAEF,sBAAsB,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC,CAAC;IAE1E,MAAM,EAAE,IAAI,EAAE,CAAC,qBAAqB,EAAE;QACrC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,WAAW;KACnB,CAAC,CAAC;IAEH,KAAK,UAAU,GAAG,CAAC,KAAsB;QACxC,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAC7B,MAAM,EAAE,IAAI,EAAE,CAAC,oBAAoB,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/D,OAAO;gBACN,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,uBAAuB,KAAK,CAAC,MAAM,EAAE;aAC9C,CAAC;QACH,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAEhD,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC9B,MAAM,EAAE,IAAI,EAAE,CAAC,mCAAmC,EAAE;gBACnD,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,KAAK,EAAE,KAAK,CAAC,KAAK;aAClB,CAAC,CAAC;YACH,OAAO;gBACN,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,0BAA0B;aACnC,CAAC;QACH,CAAC;QAED,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1B,OAAO;gBACN,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,GAAG,CAAC,OAAO;aACpB,CAAC;QACH,CAAC;QAED,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,MAAM,EAAE,IAAI,EAAE,CAAC,mBAAmB,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;QAE1E,OAAO;YACN,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,gBAAgB;YACzB,GAAG,EAAE,GAAG,CAAC,GAAG;SACZ,CAAC;IACH,CAAC;IAED,KAAK,UAAU,WAAW,CACzB,OAAiB,EACjB,SAAS,GAAG,KAAK;QAEjB,IAAI,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,CAAC,8BAA8B,EAAE;gBAC9C,MAAM,EAAE,OAAO,CAAC,KAAK;aACrB,CAAC,CAAC;YAEH,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,aAAa,CACxC;gBACC,MAAM,EAAE,OAAO,CAAC,KAAK;gBACrB,OAAO;gBACP,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,cAAc;gBACd,SAAS;aACT,EACD,OAAO,CACP,CAAC;YAEF,OAAO;gBACN,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,oBAAoB,SAAS,EAAE;gBACxC,SAAS;aACT,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,YAAY,GACjB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,EAAE,KAAK,EAAE,CAAC,6BAA6B,YAAY,EAAE,CAAC,CAAC;YAC7D,OAAO;gBACN,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,YAAY;aACrB,CAAC;QACH,CAAC;IACF,CAAC;IAED,OAAO;QACN,GAAG;QACH,WAAW;KACX,CAAC;AACH,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./types.js";
+export type { Greptor } from "./greptor.js";
+export { createGreptor } from "./greptor.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAE3B,YAAY,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC"}

package/dist/index.js

@@ -0,0 +1,3 @@
+export * from "./types.js";
+export { createGreptor } from "./greptor.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAG3B,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC"}

package/dist/llm/llm-factory.d.ts

@@ -0,0 +1,7 @@
+import OpenAI from "openai";
+export type LlmClient = {
+    client: OpenAI;
+    model: string;
+};
+export declare function createLlmClient(providerModel: string): LlmClient;
+//# sourceMappingURL=llm-factory.d.ts.map

package/dist/llm/llm-factory.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-factory.d.ts","sourceRoot":"","sources":["../../src/llm/llm-factory.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,MAAM,SAAS,GAAG;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACd,CAAC;AA+DF,wBAAgB,eAAe,CAAC,aAAa,EAAE,MAAM,GAAG,SAAS,CAShE"}

package/dist/llm/llm-factory.js

@@ -0,0 +1,53 @@
+import OpenAI from "openai";
+function parseProviderModel(providerModel) {
+    if (!providerModel.includes(":")) {
+        return { provider: "openai", model: providerModel.trim() };
+    }
+    const colonIndex = providerModel.indexOf(":");
+    const rawProvider = providerModel.substring(0, colonIndex);
+    const rawModel = providerModel.substring(colonIndex + 1);
+    if (!rawProvider || !rawModel) {
+        throw new Error(`Invalid provider:model format: ${providerModel}. Expected format: "provider:model"`);
+    }
+    const provider = rawProvider.trim().toLowerCase();
+    const model = rawModel.trim();
+    if (!provider || !model) {
+        throw new Error(`Invalid provider:model format: ${providerModel}. Both provider and model must be non-empty`);
+    }
+    return { provider, model };
+}
+function getProviderConfig(provider) {
+    const normalizedProvider = provider.toLowerCase();
+    switch (normalizedProvider) {
+        case "openai":
+            if (!process.env.OPENAI_API_KEY) {
+                throw new Error("OPENAI_API_KEY environment variable is not set.");
+            }
+            return { apiKey: process.env.OPENAI_API_KEY };
+        case "azure":
+            if (!process.env.AZURE_API_KEY || !process.env.AZURE_API_BASE_URL) {
+                throw new Error("AZURE_API_KEY or AZURE_API_BASE_URL environment variable is not set.");
+            }
+            return {
+                apiKey: process.env.AZURE_API_KEY,
+                apiUrl: process.env.AZURE_API_BASE_URL,
+            };
+        case "ollama":
+            return {
+                apiKey: "ollama",
+                apiUrl: "http://localhost:11434/v1",
+            };
+        default:
+            throw new Error(`Unsupported provider: ${provider}`);
+    }
+}
+export function createLlmClient(providerModel) {
+    const { provider, model } = parseProviderModel(providerModel);
+    const { apiKey, apiUrl } = getProviderConfig(provider);
+    const client = new OpenAI({
+        apiKey,
+        baseURL: apiUrl,
+    });
+    return { client, model };
+}
+//# sourceMappingURL=llm-factory.js.map

package/dist/llm/llm-factory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-factory.js","sourceRoot":"","sources":["../../src/llm/llm-factory.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAO5B,SAAS,kBAAkB,CAAC,aAAqB;IAIhD,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAClC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,CAAC,IAAI,EAAE,EAAE,CAAC;IAC5D,CAAC;IAED,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC9C,MAAM,WAAW,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAC3D,MAAM,QAAQ,GAAG,aAAa,CAAC,SAAS,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;IAEzD,IAAI,CAAC,WAAW,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CACd,kCAAkC,aAAa,qCAAqC,CACpF,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAClD,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;IAE9B,IAAI,CAAC,QAAQ,IAAI,CAAC,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CACd,kCAAkC,aAAa,6CAA6C,CAC5F,CAAC;IACH,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AAC5B,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAgB;IAI1C,MAAM,kBAAkB,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;IAClD,QAAQ,kBAAkB,EAAE,CAAC;QAC5B,KAAK,QAAQ;YACZ,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;YACpE,CAAC;YACD,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;QAC/C,KAAK,OAAO;YACX,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC;gBACnE,MAAM,IAAI,KAAK,CACd,sEAAsE,CACtE,CAAC;YACH,CAAC;YACD,OAAO;gBACN,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa;gBACjC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;aACtC,CAAC;QACH,KAAK,QAAQ;YACZ,OAAO;gBACN,MAAM,EAAE,QAAQ;gBAChB,MAAM,EAAE,2BAA2B;aACnC,CAAC;QACH;YACC,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,EAAE,CAAC,CAAC;IACvD,CAAC;AACF,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,aAAqB;IACpD,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,kBAAkB,CAAC,aAAa,CAAC,CAAC;IAC9D,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC;QACzB,MAAM;QACN,OAAO,EAAE,MAAM;KACf,CAAC,CAAC;IAEH,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC1B,CAAC"}

package/dist/metadata-schema/generate.d.ts

@@ -0,0 +1,3 @@
+import type { MetadataSchema } from "../types.js";
+export declare function generateMetadataSchema(topic: string, llmModel: string): Promise<MetadataSchema>;
+//# sourceMappingURL=generate.d.ts.map

package/dist/metadata-schema/generate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../src/metadata-schema/generate.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAsB,MAAM,aAAa,CAAC;AAiBtE,wBAAsB,sBAAsB,CAC3C,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,GACd,OAAO,CAAC,cAAc,CAAC,CA+BzB"}

package/dist/metadata-schema/generate.js

@@ -0,0 +1,43 @@
+import { zodResponseFormat } from "openai/helpers/zod";
+import { createLlmClient } from "../llm/llm-factory.js";
+import { ResponseSchema } from "./types.js";
+const PROMPT_TEMPLATE = (topic) => `
+You are an expert information architect designing metadata schemas that improve text search, discovery, and retrieval within a specific knowledge topic. 
+Your goal is to produce a list of 5-10 **metadata fields** that are: 
+1. **Search-relevant** — users or AI agents are likely to query or filter text by these fields. 
+2. **Domain-relevant** — reflect concepts, entities, and descriptors naturally present in this domain. 
+3. **Extractable** — values can be identified directly from text (no scoring or inferred metrics like confidence, relevance, etc.). 
+4. **Reusable** — should support both keyword search (grep/ripgrep) and structured filtering.
+
+Allowed field types: string, string[], number, number[], boolean, enum, enum[], date.
+Use array types when multiple values are expected per chunk.
+
+**TOPIC**: ${topic}
+`;
+export async function generateMetadataSchema(topic, llmModel) {
+    const { client, model } = createLlmClient(llmModel);
+    const messages = [
+        { role: "user", content: PROMPT_TEMPLATE(topic) },
+    ];
+    const completion = await client.chat.completions.parse({
+        model,
+        messages,
+        response_format: zodResponseFormat(ResponseSchema, "metadata_fields"),
+    });
+    const parsed = completion.choices[0]?.message?.parsed;
+    if (!parsed?.metadata_fields) {
+        throw new Error("Failed to parse metadata schema from LLM response");
+    }
+    return parsed.metadata_fields.map((field) => {
+        const metadataField = {
+            name: field.name,
+            type: field.type,
+            description: field.description,
+        };
+        if (Array.isArray(field.enumValues)) {
+            metadataField.enumValues = field.enumValues;
+        }
+        return metadataField;
+    });
+}
+//# sourceMappingURL=generate.js.map

package/dist/metadata-schema/generate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../src/metadata-schema/generate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAEvD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5C,MAAM,eAAe,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC;;;;;;;;;;;aAW9B,KAAK;CACjB,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC3C,KAAa,EACb,QAAgB;IAEhB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IAEpD,MAAM,QAAQ,GAAiC;QAC9C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,CAAC,KAAK,CAAC,EAAE;KACjD,CAAC;IAEF,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC;QACtD,KAAK;QACL,QAAQ;QACR,eAAe,EAAE,iBAAiB,CAAC,cAAc,EAAE,iBAAiB,CAAC;KACrE,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC;IACtD,IAAI,CAAC,MAAM,EAAE,eAAe,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACtE,CAAC;IAED,OAAO,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC3C,MAAM,aAAa,GAAuB;YACzC,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,WAAW,EAAE,KAAK,CAAC,WAAW;SAC9B,CAAC;QAEF,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;YACrC,aAAa,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;QAC7C,CAAC;QAED,OAAO,aAAa,CAAC;IACtB,CAAC,CAAC,CAAC;AACJ,CAAC"}

package/dist/metadata-schema/initialize.d.ts

@@ -0,0 +1,5 @@
+import type { Logger } from "../types.js";
+import type { MetadataSchema } from "../types.js";
+export declare const METADATA_SCHEMA_FILENAME = "metadata-schema.yaml";
+export declare function initializeMetadataSchema(baseDir: string, llmModel: string, topic: string, metadataSchema?: MetadataSchema, logger?: Logger): Promise<MetadataSchema>;
+//# sourceMappingURL=initialize.d.ts.map

package/dist/metadata-schema/initialize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"initialize.d.ts","sourceRoot":"","sources":["../../src/metadata-schema/initialize.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAIlD,eAAO,MAAM,wBAAwB,yBAAyB,CAAC;AAc/D,wBAAsB,wBAAwB,CAC7C,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,cAAc,CAAC,EAAE,cAAc,EAC/B,MAAM,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,cAAc,CAAC,CA4BzB"}