mcp-searchable 1.0.2

package/README.md

@@ -0,0 +1,312 @@
+# TypeScript Bootstrap Template
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Node.js CI](https://github.com/templ-project/typescript/actions/workflows/ci.yml/badge.svg)](https://github.com/templ-project/typescript/actions/workflows/ci.yml)
+[![Contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/templ-project/typescript/issues)
+
+> A modern TypeScript project template with ESM, testing, linting, and quality tools built-in.
+
+- [TypeScript Bootstrap Template](#typescript-bootstrap-template)
+  - [Quick Start](#quick-start)
+    - [Bootstrap Options](#bootstrap-options)
+  - [What's Included](#whats-included)
+  - [Common Commands](#common-commands)
+    - [Using npm](#using-npm)
+    - [Using mise tasks (Recommended)](#using-taskfile-recommended)
+  - [Requirements](#requirements)
+  - [Setup Development Environment](#setup-development-environment)
+  - [Project Structure](#project-structure)
+  - [Building](#building)
+  - [Testing](#testing)
+  - [Code Quality](#code-quality)
+    - [Pre-commit Hooks](#pre-commit-hooks)
+  - [Configuration](#configuration)
+  - [Using as a Library](#using-as-a-library)
+  - [CI/CD Pipeline](#cicd-pipeline)
+  - [License](#license)
+  - [Support](#support)
+
+## Quick Start
+
+**Bootstrap a new project:**
+
+```bash
+npx --yes --package=github:templ-project/typescript bootstrap ./my-project
+cd my-project
+npm install
+npm test
+```
+
+That's it! You now have a fully configured TypeScript project.
+
+## MCP Search Providers
+
+The `web_search` MCP tool supports multiple providers via the optional `provider` argument:
+
+- `duckduckgo` (default)
+- `google` (Google Custom Search JSON API)
+- `bing` (Bing Web Search API)
+- `brave` (Brave Search API)
+
+### Tool input
+
+```json
+{
+  "query": "model context protocol",
+  "provider": "google",
+  "limit": 5
+}
+```
+
+### Required environment variables
+
+- `GOOGLE_API_KEY` and `GOOGLE_CSE_ID` for `provider=google`
+- `BING_API_KEY` for `provider=bing`
+- `BRAVE_API_KEY` for `provider=brave`
+
+Optional:
+
+- `BING_API_ENDPOINT` (override default Bing endpoint)
+
+You can copy `.env.example` and fill in your provider keys.
+
+### Bootstrap Options
+
+```bash
+# Bootstrap with only ESM and CJS builds (no browser builds)
+npx --yes --package=github:templ-project/typescript bootstrap --target esm,cjs ./my-project
+
+# Bootstrap as part of a monorepo (removes .husky, .github)
+npx --yes --package=github:templ-project/typescript bootstrap --part-of-monorepo ./packages/my-lib
+
+# Show all available options
+npx --yes --package=github:templ-project/typescript bootstrap --help
+```
+
+See [.npx-install/README.md](.npx-install/README.md) for detailed bootstrap documentation.
+
+## What's Included
+
+| Feature                 | Tool                                                                                                    | Description                        |
+| ----------------------- | ------------------------------------------------------------------------------------------------------- | ---------------------------------- |
+| **Language**            | [TypeScript 5.9](https://www.typescriptlang.org/)                                                       | Type-safe JavaScript               |
+| **Runtime**             | [Node.js 22+](https://nodejs.org/)                                                                      | Modern JavaScript runtime          |
+| **Execution**           | [tsx](https://github.com/privatenumber/tsx)                                                             | TypeScript execution without build |
+| **Module System**       | ESM                                                                                                     | Native ES Modules                  |
+| **Build System**        | [tsc](https://www.typescriptlang.org/) + [esbuild](https://esbuild.github.io/)                          | Type-check + fast bundling         |
+| **Test Framework**      | [Vitest](https://vitest.dev/)                                                                           | Fast unit testing with V8 coverage |
+| **Linting**             | [ESLint](https://eslint.org/)                                                                           | Code quality with TypeScript rules |
+| **Formatting**          | [Prettier](https://prettier.io/)                                                                        | Consistent code formatting         |
+| **Documentation**       | [MkDocs](https://www.mkdocs.org/) + [Material for MkDocs](https://squidfunk.github.io/mkdocs-material/) | Project documentation site         |
+| **Task Runner**         | [mise tasks](https://mise.jdx.dev/tasks/)                                                               | Modern build automation            |
+| **Tool Management**     | [mise](https://mise.jdx.dev/)                                                                           | Isolated development environment   |
+| **Pre-commit Hooks**    | [Husky](https://typicode.github.io/husky/) + [lint-staged](https://github.com/okonet/lint-staged)       | Automatic validation               |
+| **Duplicate Detection** | [jscpd](https://github.com/kucherenko/jscpd)                                                            | Copy-paste detector                |
+| **CI/CD**               | GitHub Actions                                                                                          | Multi-platform testing & releases  |
+
+## Common Commands
+
+### Using npm
+
+```bash
+npm start              # Run the app (via tsx)
+npm test               # Run tests
+npm run test:coverage  # Run tests with coverage
+npm run lint           # Lint and auto-fix
+npm run format         # Format code with Prettier
+npm run build          # Build for production
+npm run docs           # Build MkDocs site (strict)
+npm run validate       # Run all quality checks
+```
+
+### Using mise tasks (Recommended)
+
+```bash
+# === Development ===
+mise run run              # Run the application (via node)
+mise run format            # Format all code (Prettier)
+mise run format:check      # Check formatting without fixing
+mise run lint              # Lint all code (ESLint + TypeScript)
+mise run lint:check        # Check all without fixing
+mise run duplicate-check   # Check for duplicate code
+mise run docs              # Build MkDocs site (strict)
+mise run docs:serve        # Serve documentation locally
+mise run deps:sync         # Install all dependencies (mise, npm)
+mise run deps:refresh      # Update all dependencies
+mise run deps:clean        # Remove all dependencies
+```
+
+## Requirements
+
+- [mise](https://mise.jdx.dev/) - Tool version management (installs everything else)
+
+**Automatically installed via mise:**
+
+- Node.js 22+
+- Python 3.11+ (for helper scripts)
+- ShellCheck (shell script linting)
+- PowerShell Core (cross-platform PowerShell)
+
+## Setup Development Environment
+
+```bash
+# Install mise (if not already installed)
+# Linux/macOS:
+curl https://mise.run | sh
+
+# Windows (PowerShell):
+winget install jdx.mise
+# or: choco install mise
+
+# Install project tools and node modules
+
+
+# Clone and setup
+git clone https://github.com/templ-project/typescript.git my-project
+cd my-project
+
+# Install all dependencies
+mise run deps:sync
+
+# Verify setup
+mise run validate
+```
+
+## Project Structure
+
+```text
+├── .github/
+│   └── workflows/        # CI/CD pipelines
+├── .husky/               # Git hooks
+├── .scripts/             # Build/lint helper scripts
+
+├── src/
+│   ├── index.ts          # Main entry point
+│   └── lib/
+│       ├── greeter.ts    # Example module with TypeScript types
+│       └── greeter.spec.ts # Unit tests (co-located)
+├── dist/                 # Build output (gitignored)
+├── docs/                 # MkDocs source pages
+├── mise.toml             # Configuration and tasks
+├── .mise.toml            # Tool versions & hooks
+├── package.json          # Node.js dependencies
+├── tsconfig.json         # TypeScript configuration
+├── vitest.config.ts      # Test configuration
+├── eslint.config.mjs     # ESLint configuration
+├── prettier.config.mjs   # Prettier configuration
+└── mkdocs.yml            # MkDocs configuration
+```
+
+## Building
+
+The template uses a two-step build process:
+
+1. **tsc** - Type-checks and emits declaration files (`.d.ts`)
+2. **esbuild** - Bundles for multiple targets
+
+```bash
+npm run build
+# or: npm run build
+```
+
+Outputs:
+
+- `dist/index.js` - Node.js ESM build artifact
+- `dist/index.d.ts` - TypeScript declaration file
+
+## Testing
+
+Tests are co-located with source files using the `.spec.ts` suffix:
+
+```typescript
+// src/lib/greeter.spec.ts
+import { describe, it, expect } from "vitest";
+import { hello } from "./greeter";
+
+describe("hello", () => {
+  it("returns greeting", () => {
+    expect(hello("World")).toBe("Hello, World!");
+  });
+});
+```
+
+Run tests:
+
+```bash
+npm run test                # Run all tests
+npm run test:coverage       # Run with coverage report
+```
+
+## Code Quality
+
+### Pre-commit Hooks
+
+Every commit is validated with:
+
+- Code formatting (Prettier)
+- Linting (ESLint)
+- Type checking (TypeScript)
+- Unit tests (Vitest)
+
+CI runs additional checks:
+
+- Test coverage
+- Duplicate code detection
+- License compliance
+
+Configure hooks in:
+
+- `.husky/pre-commit` - Hook script
+- `.lintstagedrc.yml` - File patterns and commands
+
+## Configuration
+
+All configuration uses shared packages for consistency:
+
+| File                  | Purpose                                               |
+| --------------------- | ----------------------------------------------------- |
+| `.mise.toml`          | Tool versions (Node, Python, ShellCheck)              |
+| `mise.toml`           | Configuration and tasks                               |
+| `tsconfig.json`       | TypeScript config (extends `@templ-project/tsconfig`) |
+| `eslint.config.mjs`   | ESLint config (uses `@templ-project/eslint`)          |
+| `prettier.config.mjs` | Prettier config (uses `@templ-project/prettier`)      |
+| `vitest.config.ts`    | Vitest test configuration                             |
+| `mkdocs.yml`          | MkDocs site navigation and theme settings             |
+| `.jscpd.json`         | Duplicate detection settings                          |
+| `.licensee.json`      | License compliance settings                           |
+
+## Using as a Library
+
+```typescript
+// ES Modules (recommended)
+import { hello, Greeter } from "@templ-project/typescript-template";
+
+const greeting = hello("World");
+console.log(greeting); // "Hello, World!"
+```
+
+```javascript
+// ESM from built output
+import { hello } from "./dist/index.js";
+```
+
+## CI/CD Pipeline
+
+The GitHub Actions pipeline runs on **Linux, macOS, and Windows**:
+
+| Workflow         | Trigger                 | Jobs                               |
+| ---------------- | ----------------------- | ---------------------------------- |
+| `ci.yml`         | Push/PR to main/develop | Matrix orchestrator                |
+| `ci.quality.yml` | Called by ci.yml        | lint, test, build, duplicate-check |
+| `ci.version.yml` | Push to main            | Semantic version bump              |
+| `ci.release.yml` | After version bump      | Create GitHub release              |
+
+## License
+
+MIT © [Templ Project](https://github.com/templ-project)
+
+## Support
+
+- [Report Issues](https://github.com/templ-project/typescript/issues)
+- [Read the Docs](https://github.com/templ-project/typescript#readme)
+- [Star on GitHub](https://github.com/templ-project/typescript)

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":""}

package/dist/index.js

@@ -0,0 +1,185 @@
+#!/usr/bin/env node
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
+import { WebStorage } from './lib/storage.js';
+import { OllamaAskProvider } from './lib/web-ask.js';
+import { WebFetcher } from './lib/web-fetch.js';
+import { createSearchProvider } from './lib/web-search.js';
+const SEARCH_PROVIDERS = ['duckduckgo', 'google', 'bing', 'brave'];
+const webFetcher = new WebFetcher();
+const webStorage = new WebStorage('.web_stash.db');
+const askProvider = new OllamaAskProvider(webStorage);
+const server = new Server({
+    name: 'smart-web-extraction',
+    version: '1.0.0',
+}, {
+    capabilities: {
+        tools: {},
+    },
+});
+const SearchInputSchema = {
+    type: 'object',
+    properties: {
+        query: { type: 'string', description: 'Search query' },
+        provider: {
+            type: 'string',
+            enum: SEARCH_PROVIDERS,
+            description: 'Search provider (default: duckduckgo)',
+        },
+        limit: { type: 'number', description: 'Max results (default: 5)' },
+    },
+    required: ['query'],
+};
+function parseRequiredString(value, fieldName) {
+    if (typeof value !== 'string') {
+        throw new Error(`Invalid ${fieldName}: expected a string`);
+    }
+    const normalized = value.trim();
+    if (!normalized) {
+        throw new Error(`Invalid ${fieldName}: cannot be empty`);
+    }
+    return normalized;
+}
+function parseOptionalPositiveLimit(value, defaultValue) {
+    if (value === undefined) {
+        return defaultValue;
+    }
+    if (typeof value !== 'number' || !Number.isFinite(value) || !Number.isInteger(value) || value <= 0) {
+        throw new Error('Invalid limit: expected a positive integer');
+    }
+    return value;
+}
+function parseProvider(value) {
+    if (value === undefined) {
+        return 'duckduckgo';
+    }
+    if (typeof value !== 'string') {
+        throw new Error('Invalid provider: expected a string');
+    }
+    const provider = value.trim();
+    if (!SEARCH_PROVIDERS.includes(provider)) {
+        throw new Error(`Invalid provider: ${provider}. Supported providers: ${SEARCH_PROVIDERS.join(', ')}`);
+    }
+    return provider;
+}
+server.setRequestHandler(ListToolsRequestSchema, async () => {
+    return {
+        tools: [
+            {
+                name: 'web_search',
+                description: 'Search the web using supported providers (DuckDuckGo, Google, Bing, Brave).',
+                inputSchema: SearchInputSchema,
+            },
+            {
+                name: 'web_fetch',
+                description: 'Fetch and extract markdown content from a URL.',
+                inputSchema: {
+                    type: 'object',
+                    properties: {
+                        url: { type: 'string', description: 'URL to fetch' },
+                    },
+                    required: ['url'],
+                },
+            },
+            {
+                name: 'web_stash',
+                description: 'Store markdown content into local SQLite database.',
+                inputSchema: {
+                    type: 'object',
+                    properties: {
+                        url: { type: 'string', description: 'Source URL' },
+                        title: { type: 'string', description: 'Title of the page' },
+                        content: { type: 'string', description: 'Markdown content' },
+                    },
+                    required: ['url', 'title', 'content'],
+                },
+            },
+            {
+                name: 'web_grep',
+                description: 'Search stashed web pages using FTS5.',
+                inputSchema: SearchInputSchema,
+            },
+            {
+                name: 'web_ask',
+                description: 'Ask a question using a local Ollama model and context from stashed web pages.',
+                inputSchema: {
+                    type: 'object',
+                    properties: {
+                        question: { type: 'string', description: 'The question to ask' },
+                        limit: { type: 'number', description: 'Number of sources to retrieve (default: 3)' },
+                    },
+                    required: ['question'],
+                },
+            },
+        ],
+    };
+});
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+    const { name, arguments: args } = request.params;
+    try {
+        switch (name) {
+            case 'web_search': {
+                const query = parseRequiredString(args?.query, 'query');
+                const provider = parseProvider(args?.provider);
+                const limit = parseOptionalPositiveLimit(args?.limit, 5);
+                const searchProvider = createSearchProvider(provider);
+                const results = await searchProvider.search(query, limit);
+                return {
+                    content: [{ type: 'text', text: JSON.stringify(results, null, 2) }],
+                };
+            }
+            case 'web_fetch': {
+                const url = parseRequiredString(args?.url, 'url');
+                const result = await webFetcher.fetch(url);
+                return {
+                    content: [{ type: 'text', text: JSON.stringify(result, null, 2) }],
+                };
+            }
+            case 'web_stash': {
+                const url = parseRequiredString(args?.url, 'url');
+                const title = parseRequiredString(args?.title, 'title');
+                const content = parseRequiredString(args?.content, 'content');
+                webStorage.stash({ url, title, content });
+                return {
+                    content: [{ type: 'text', text: `Successfully stashed ${url}` }],
+                };
+            }
+            case 'web_grep': {
+                const query = parseRequiredString(args?.query, 'query');
+                const limit = parseOptionalPositiveLimit(args?.limit, 5);
+                const results = webStorage.grep(query, limit);
+                return {
+                    content: [{ type: 'text', text: JSON.stringify(results, null, 2) }],
+                };
+            }
+            case 'web_ask': {
+                const question = parseRequiredString(args?.question, 'question');
+                const limit = parseOptionalPositiveLimit(args?.limit, 3);
+                const result = await askProvider.ask(question, limit);
+                return {
+                    content: [{ type: 'text', text: `Answer:\n${result.answer}\n\nSources:\n${result.contextUrls.join('\n')}` }],
+                };
+            }
+            default:
+                throw new Error(`Unknown tool: ${name}`);
+        }
+    }
+    catch (error) {
+        const err = error;
+        return {
+            content: [{ type: 'text', text: `Error executing tool ${name}: ${err.message}` }],
+            isError: true,
+        };
+    }
+});
+async function run() {
+    const transport = new StdioServerTransport();
+    await server.connect(transport);
+    console.error('Smart Web Extraction MCP Server running on stdio');
+}
+run().catch((error) => {
+    console.error('Fatal error running server:', error);
+    process.exit(1);
+});
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AACnG,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,oBAAoB,EAA2B,MAAM,qBAAqB,CAAC;AAEpF,MAAM,gBAAgB,GAAyB,CAAC,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;AAGzF,MAAM,UAAU,GAAG,IAAI,UAAU,EAAE,CAAC;AACpC,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,eAAe,CAAC,CAAC;AACnD,MAAM,WAAW,GAAG,IAAI,iBAAiB,CAAC,UAAU,CAAC,CAAC;AAEtD,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB;IACE,IAAI,EAAE,sBAAsB;IAC5B,OAAO,EAAE,OAAO;CACjB,EACD;IACE,YAAY,EAAE;QACZ,KAAK,EAAE,EAAE;KACV;CACF,CACF,CAAC;AAEF,MAAM,iBAAiB,GAAG;IACxB,IAAI,EAAE,QAAQ;IACd,UAAU,EAAE;QACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE;QACtD,QAAQ,EAAE;YACR,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,gBAAgB;YACtB,WAAW,EAAE,uCAAuC;SACrD;QACD,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,0BAA0B,EAAE;KACnE;IACD,QAAQ,EAAE,CAAC,OAAO,CAAC;CACpB,CAAC;AAEF,SAAS,mBAAmB,CAAC,KAAc,EAAE,SAAiB;IAC5D,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,WAAW,SAAS,qBAAqB,CAAC,CAAC;IAC7D,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,WAAW,SAAS,mBAAmB,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,0BAA0B,CAAC,KAAc,EAAE,YAAoB;IACtE,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QACnG,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,EAAwB,CAAC;IACpD,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,0BAA0B,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxG,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAGD,MAAM,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE;IAC1D,OAAO;QACL,KAAK,EAAE;YACL;gBACE,IAAI,EAAE,YAAY;gBAClB,WAAW,EAAE,6EAA6E;gBAC1F,WAAW,EAAE,iBAAiB;aAC/B;YACD;gBACE,IAAI,EAAE,WAAW;gBACjB,WAAW,EAAE,gDAAgD;gBAC7D,WAAW,EAAE;oBACX,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE;wBACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE;qBACrD;oBACD,QAAQ,EAAE,CAAC,KAAK,CAAC;iBAClB;aACF;YACD;gBACE,IAAI,EAAE,WAAW;gBACjB,WAAW,EAAE,oDAAoD;gBACjE,WAAW,EAAE;oBACX,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE;wBACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY,EAAE;wBAClD,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,mBAAmB,EAAE;wBAC3D,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE;qBAC7D;oBACD,QAAQ,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC;iBACtC;aACF;YACD;gBACE,IAAI,EAAE,UAAU;gBAChB,WAAW,EAAE,sCAAsC;gBACnD,WAAW,EAAE,iBAAiB;aAC/B;YACD;gBACE,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,+EAA+E;gBAC5F,WAAW,EAAE;oBACX,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE;wBACV,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qBAAqB,EAAE;wBAChE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,4CAA4C,EAAE;qBACrF;oBACD,QAAQ,EAAE,CAAC,UAAU,CAAC;iBACvB;aACF;SACF;KACF,CAAC;AACJ,CAAC,CAAC,CAAC;AAGH,MAAM,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;IAChE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAEjD,IAAI,CAAC;QACH,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;gBACxD,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;gBAC/C,MAAM,KAAK,GAAG,0BAA0B,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;gBAEzD,MAAM,cAAc,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;gBACtD,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;gBAC1D,OAAO;oBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;iBACpE,CAAC;YACJ,CAAC;YAED,KAAK,WAAW,CAAC,CAAC,CAAC;gBACjB,MAAM,GAAG,GAAG,mBAAmB,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;gBAClD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC3C,OAAO;oBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;iBACnE,CAAC;YACJ,CAAC;YAED,KAAK,WAAW,CAAC,CAAC,CAAC;gBACjB,MAAM,GAAG,GAAG,mBAAmB,CAAC,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;gBAClD,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;gBACxD,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;gBAC9D,UAAU,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;gBAC1C,OAAO;oBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,wBAAwB,GAAG,EAAE,EAAE,CAAC;iBACjE,CAAC;YACJ,CAAC;YAED,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;gBACxD,MAAM,KAAK,GAAG,0BAA0B,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;gBACzD,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;gBAC9C,OAAO;oBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;iBACpE,CAAC;YACJ,CAAC;YAED,KAAK,SAAS,CAAC,CAAC,CAAC;gBACf,MAAM,QAAQ,GAAG,mBAAmB,CAAC,IAAI,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;gBACjE,MAAM,KAAK,GAAG,0BAA0B,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;gBAEzD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;gBAEtD,OAAO;oBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,MAAM,CAAC,MAAM,iBAAiB,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;iBAC7G,CAAC;YACJ,CAAC;YAED;gBACE,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,KAAc,CAAC;QAC3B,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,wBAAwB,IAAI,KAAK,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;YACjF,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CAAC,CAAC;AAGH,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,OAAO,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;AACpE,CAAC;AAED,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACpB,OAAO,CAAC,KAAK,CAAC,6BAA6B,EAAE,KAAK,CAAC,CAAC;IACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/lib/storage.d.ts

@@ -0,0 +1,17 @@
+import { WebFetchResult } from './web-fetch.js';
+export interface GrepResult {
+    url: string;
+    title: string;
+    snippet: string;
+    rank: number;
+}
+export declare class WebStorage {
+    private db;
+    constructor(dbPath?: string);
+    private initSchema;
+    stash(page: WebFetchResult): void;
+    grep(query: string, limit?: number): GrepResult[];
+    getPage(url: string): WebFetchResult | null;
+    close(): void;
+}
+//# sourceMappingURL=storage.d.ts.map

package/dist/lib/storage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../../src/lib/storage.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,EAAE,CAAe;gBAEb,MAAM,GAAE,MAAmB;IAKvC,OAAO,CAAC,UAAU;IA0ClB,KAAK,CAAC,IAAI,EAAE,cAAc,GAAG,IAAI;IAsBjC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,GAAG,UAAU,EAAE;IA+BrD,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,cAAc,GAAG,IAAI;IA2B3C,KAAK,IAAI,IAAI;CAGd"}

package/dist/lib/storage.js

@@ -0,0 +1,105 @@
+import Database from 'better-sqlite3';
+export class WebStorage {
+    db;
+    constructor(dbPath = ':memory:') {
+        this.db = new Database(dbPath);
+        this.initSchema();
+    }
+    initSchema() {
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS pages (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        url TEXT UNIQUE NOT NULL,
+        title TEXT,
+        content TEXT NOT NULL,
+        excerpt TEXT,
+        site_name TEXT,
+        created_at INTEGER NOT NULL
+      );
+    `);
+        this.db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5(
+        title,
+        content,
+        content='pages',
+        content_rowid='id'
+      );
+    `);
+        this.db.exec(`
+      CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN
+        INSERT INTO pages_fts(rowid, title, content) VALUES (new.id, new.title, new.content);
+      END;
+      
+      CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN
+        INSERT INTO pages_fts(pages_fts, rowid, title, content) VALUES('delete', old.id, old.title, old.content);
+      END;
+      
+      CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN
+        INSERT INTO pages_fts(pages_fts, rowid, title, content) VALUES('delete', old.id, old.title, old.content);
+        INSERT INTO pages_fts(rowid, title, content) VALUES (new.id, new.title, new.content);
+      END;
+    `);
+    }
+    stash(page) {
+        const stmt = this.db.prepare(`
+      INSERT INTO pages (url, title, content, excerpt, site_name, created_at)
+      VALUES (@url, @title, @content, @excerpt, @siteName, @createdAt)
+      ON CONFLICT(url) DO UPDATE SET
+        title = excluded.title,
+        content = excluded.content,
+        excerpt = excluded.excerpt,
+        site_name = excluded.site_name,
+        created_at = excluded.created_at
+    `);
+        stmt.run({
+            url: page.url,
+            title: page.title,
+            content: page.content,
+            excerpt: page.excerpt || null,
+            siteName: page.siteName || null,
+            createdAt: Date.now(),
+        });
+    }
+    grep(query, limit = 10) {
+        const stmt = this.db.prepare(`
+      SELECT 
+        pages.url,
+        pages.title,
+        snippet(pages_fts, 1, '<b>', '</b>', '...', 64) as snippet,
+        pages_fts.rank
+      FROM pages_fts
+      JOIN pages ON pages.id = pages_fts.rowid
+      WHERE pages_fts MATCH @query
+      ORDER BY pages_fts.rank
+      LIMIT @limit
+    `);
+        const rows = stmt.all({ query, limit });
+        return rows.map((row) => ({
+            url: row.url,
+            title: row.title,
+            snippet: row.snippet,
+            rank: row.rank,
+        }));
+    }
+    getPage(url) {
+        const stmt = this.db.prepare(`
+      SELECT url, title, content, excerpt, site_name as siteName
+      FROM pages
+      WHERE url = @url
+    `);
+        const row = stmt.get({ url });
+        if (!row)
+            return null;
+        return {
+            url: row.url,
+            title: row.title,
+            content: row.content,
+            excerpt: row.excerpt,
+            siteName: row.siteName,
+        };
+    }
+    close() {
+        this.db.close();
+    }
+}
+//# sourceMappingURL=storage.js.map

package/dist/lib/storage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.js","sourceRoot":"","sources":["../../src/lib/storage.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAWtC,MAAM,OAAO,UAAU;IACb,EAAE,CAAe;IAEzB,YAAY,SAAiB,UAAU;QACrC,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC,UAAU,EAAE,CAAC;IACpB,CAAC;IAEO,UAAU;QAEhB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;KAUZ,CAAC,CAAC;QAIH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;KAOZ,CAAC,CAAC;QAGH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;KAaZ,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,IAAoB;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;;KAS5B,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC;YACP,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI;YAC7B,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI;YAC/B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC,CAAC;IACL,CAAC;IAED,IAAI,CAAC,KAAa,EAAE,QAAgB,EAAE;QAGpC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;KAW5B,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAKpC,CAAC;QAEH,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxB,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC,CAAC,CAAC;IACN,CAAC;IAED,OAAO,CAAC,GAAW;QACjB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;KAI5B,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAQf,CAAC;QACd,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,OAAO;YACL,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ;SACvB,CAAC;IACJ,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACF"}

package/dist/lib/web-ask.d.ts

@@ -0,0 +1,13 @@
+import { WebStorage } from './storage.js';
+export interface AskResult {
+    answer: string;
+    contextUrls: string[];
+}
+export declare class OllamaAskProvider {
+    private storage;
+    private ollamaUrl;
+    private model;
+    constructor(storage: WebStorage, ollamaUrl?: string, model?: string);
+    ask(question: string, limit?: number): Promise<AskResult>;
+}
+//# sourceMappingURL=web-ask.d.ts.map

package/dist/lib/web-ask.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"web-ask.d.ts","sourceRoot":"","sources":["../../src/lib/web-ask.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAc,MAAM,cAAc,CAAC;AAEtD,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qBAAa,iBAAiB;IAE1B,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,KAAK;gBAFL,OAAO,EAAE,UAAU,EACnB,SAAS,GAAE,MAA2D,EACtE,KAAK,GAAE,MAA6C;IAGxD,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,GAAE,MAAU,GAAG,OAAO,CAAC,SAAS,CAAC;CAiFnE"}