@sylphx/lookout 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +127 -0
- package/bin/lookout +11 -0
- package/docs/BRAND_PUBLISH.md +36 -0
- package/docs/COMPETITIVE.md +27 -0
- package/docs/EVIDENCE_CONTRACT.md +21 -0
- package/docs/IPPB.md +16 -0
- package/docs/POSITIONING.md +42 -0
- package/docs/PRODUCT_INDEPENDENCE.md +13 -0
- package/docs/PUBLISH.md +23 -0
- package/docs/TOOL_SURFACE.md +19 -0
- package/package.json +58 -0
- package/server.json +22 -0
- package/src/cache.ts +129 -0
- package/src/cli.ts +106 -0
- package/src/crawl.ts +204 -0
- package/src/doctor.ts +73 -0
- package/src/engine.ts +427 -0
- package/src/extract.ts +274 -0
- package/src/fetch.ts +145 -0
- package/src/mcp.ts +118 -0
- package/src/research.ts +98 -0
- package/src/robots.ts +97 -0
- package/src/sdk.ts +59 -0
- package/src/search.ts +356 -0
- package/src/sitemap.ts +13 -0
- package/src/ssrf.ts +69 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 SylphxAI
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# Lookout
|
|
2
|
+
|
|
3
|
+
### The web from your machine.
|
|
4
|
+
|
|
5
|
+
**Lookout** is a local-first **web instrument** for agents and apps: **search, fetch, extract, cache** with citeable excerpts — **no required API key**, **tiny default install**, same API on **SDK · CLI · MCP**.
|
|
6
|
+
|
|
7
|
+
Primary competitive anchor: [wigolo](https://github.com/KnockOutEZ/wigolo) (learn multi-surface + honesty; **do not** require multi-GB browser/model warmup).
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
## Product docs
|
|
12
|
+
|
|
13
|
+
| Doc | Purpose |
|
|
14
|
+
| --- | --- |
|
|
15
|
+
| [docs/POSITIONING.md](docs/POSITIONING.md) | Strategic positioning |
|
|
16
|
+
| [docs/COMPETITIVE.md](docs/COMPETITIVE.md) | Peer anchors and wedge |
|
|
17
|
+
| [docs/EVIDENCE_CONTRACT.md](docs/EVIDENCE_CONTRACT.md) | Evidence = result contract |
|
|
18
|
+
| [docs/TOOL_SURFACE.md](docs/TOOL_SURFACE.md) | Few clear tools policy |
|
|
19
|
+
| [docs/PRODUCT_INDEPENDENCE.md](docs/PRODUCT_INDEPENDENCE.md) | This repo is SSOT |
|
|
20
|
+
| [docs/IPPB.md](docs/IPPB.md) | Independent public product bar |
|
|
21
|
+
| [docs/PUBLISH.md](docs/PUBLISH.md) | npm/git publish status |
|
|
22
|
+
|
|
23
|
+
## Agent skill surface
|
|
24
|
+
|
|
25
|
+
See [`skills/lookout/SKILL.md`](./skills/lookout/SKILL.md).
|
|
26
|
+
|
|
27
|
+
## Why Lookout (vs multi-GB “local web” stacks)
|
|
28
|
+
|
|
29
|
+
| Typical agent web path | Lookout |
|
|
30
|
+
| --- | --- |
|
|
31
|
+
| Paid search API keys | Default **public adapters**, $0/query |
|
|
32
|
+
| Multi-GB browser + model warmup | **Tiny default** — HTML search/fetch/extract |
|
|
33
|
+
| Opaque snippets | **Cite spans** + extract route + warnings |
|
|
34
|
+
| One mega-tool | **Clear tools**: search · fetch · extract · cache (+ advanced crawl) |
|
|
35
|
+
|
|
36
|
+
Primary competitive anchor: [wigolo](https://github.com/KnockOutEZ/wigolo) — learn multi-surface honesty; **do not** require heavy browser/model installs on the default path.
|
|
37
|
+
|
|
38
|
+
Evidence is a **result contract** (spans, routes, warnings) — not a tool named `evidence_first`.
|
|
39
|
+
|
|
40
|
+
## Install (30 seconds)
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
npm install -g @sylphx/lookout
|
|
44
|
+
# brand bin
|
|
45
|
+
lookout doctor
|
|
46
|
+
lookout search "model context protocol"
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Claude Code:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
claude mcp add lookout -- npx @sylphx/lookout
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Install (dev)
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
git clone https://github.com/SylphxAI/lookout.git
|
|
59
|
+
cd lookout
|
|
60
|
+
bun install
|
|
61
|
+
./bin/lookout tools
|
|
62
|
+
./bin/lookout extract # via engine with -- see CLI
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
# CLI
|
|
67
|
+
./bin/lookout doctor
|
|
68
|
+
./bin/lookout tools
|
|
69
|
+
./bin/lookout search "local-first agents"
|
|
70
|
+
./bin/lookout fetch https://example.com
|
|
71
|
+
./bin/lookout extract https://example.com
|
|
72
|
+
./bin/lookout cache stats
|
|
73
|
+
./bin/lookout crawl https://example.com --depth 0 --pages 2
|
|
74
|
+
|
|
75
|
+
# MCP (stdio)
|
|
76
|
+
bun src/mcp.ts
|
|
77
|
+
# or: ./bin/lookout mcp
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
## SDK
|
|
81
|
+
|
|
82
|
+
```ts
|
|
83
|
+
import { Lookout } from '@sylphx/lookout'
|
|
84
|
+
|
|
85
|
+
const lookout = Lookout.create()
|
|
86
|
+
const search = await lookout.search('model context protocol')
|
|
87
|
+
const page = await lookout.fetch('https://example.com')
|
|
88
|
+
const extracted = await lookout.extract({ url: 'https://example.com' })
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Tools (clear, not merged)
|
|
92
|
+
|
|
93
|
+
| Tool | Job |
|
|
94
|
+
| --- | --- |
|
|
95
|
+
| `web_search` | Public adapters (DuckDuckGo HTML + Wikipedia + npm registry + HN Algolia), rank fusion, query-term boost, host filters |
|
|
96
|
+
| `web_fetch` | SSRF-safe HTTP(S) fetch, redirects, size limits, cite spans |
|
|
97
|
+
| `web_extract` | Title, description, JSON-LD, tables, spans from HTML/URL |
|
|
98
|
+
| `web_cache` | Local disk cache query/stats/clear/prune (`LOOKOUT_CACHE_DIR`, optional max age) |
|
|
99
|
+
| `web_crawl` *(advanced)* | Same-origin, depth-limited crawl (not a full-site crawler) |
|
|
100
|
+
| `web_research` *(advanced)* | Search then fetch/extract top pages with citeable excerpts |
|
|
101
|
+
|
|
102
|
+
## Environment
|
|
103
|
+
|
|
104
|
+
| Var | Purpose |
|
|
105
|
+
| --- | --- |
|
|
106
|
+
| `LOOKOUT_CACHE_DIR` | Cache directory (default `~/.cache/lookout`) |
|
|
107
|
+
| `LOOKOUT_CACHE_MAX_AGE_MS` | Optional max age for search/fetch cache hits |
|
|
108
|
+
| `LOOKOUT_LIVE=1` | Enable optional live network tests |
|
|
109
|
+
| `LOOKOUT_USER_AGENT` | Optional User-Agent override for fetch |
|
|
110
|
+
| `LOOKOUT_FETCH_TIMEOUT_MS` | Optional fetch timeout (ms) |
|
|
111
|
+
| `LOOKOUT_FETCH_MAX_BYTES` | Optional max response body bytes |
|
|
112
|
+
|
|
113
|
+
## Safety
|
|
114
|
+
|
|
115
|
+
- Private IPv4/IPv6, localhost, link-local metadata hosts **denied**
|
|
116
|
+
- Only `http`/`https`
|
|
117
|
+
- Redirect cap + response size cap
|
|
118
|
+
|
|
119
|
+
## Family
|
|
120
|
+
|
|
121
|
+
Citra · Iris · Cue · Prism · Spine · **Lookout**
|
|
122
|
+
|
|
123
|
+
Company portfolio knowledge (docs only — not a product monorepo):
|
|
124
|
+
This repository is product SSOT. Sibling agent tools are separate repositories and are not vendored here.
|
|
125
|
+
## License
|
|
126
|
+
|
|
127
|
+
MIT
|
package/bin/lookout
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
4
|
+
if command -v bun >/dev/null 2>&1; then
|
|
5
|
+
exec bun "$ROOT/src/cli.ts" "$@"
|
|
6
|
+
fi
|
|
7
|
+
if command -v node >/dev/null 2>&1; then
|
|
8
|
+
exec node --experimental-strip-types "$ROOT/src/cli.ts" "$@"
|
|
9
|
+
fi
|
|
10
|
+
echo "lookout requires bun or node>=22" >&2
|
|
11
|
+
exit 1
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Lookout — brand npm publish readiness
|
|
2
|
+
|
|
3
|
+
**Publish authority:** this repository only (not a multi-product monorepo).
|
|
4
|
+
|
|
5
|
+
| Field | Value |
|
|
6
|
+
| --- | --- |
|
|
7
|
+
| Brand | **Lookout** |
|
|
8
|
+
| Transitional npm id | `@sylphx/lookout` |
|
|
9
|
+
| Target brand npm id | `@sylphx/lookout` |
|
|
10
|
+
| Brand bin | `lookout` |
|
|
11
|
+
| Marketplace title | Lookout (`server.json`) |
|
|
12
|
+
|
|
13
|
+
## Current policy
|
|
14
|
+
|
|
15
|
+
1. Ship/publish `@sylphx/lookout` from this repo (existing CI/release train).
|
|
16
|
+
2. Optional second publish of `@sylphx/lookout` **from this same repo** (same artifacts, renamed package.json at pack time).
|
|
17
|
+
3. Never publish brand packages from any other repository.
|
|
18
|
+
|
|
19
|
+
## Dry-run (no npm auth required)
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
# Transitional package as configured
|
|
23
|
+
npm pack --dry-run
|
|
24
|
+
# Or product-specific brand pack plan script when present:
|
|
25
|
+
# bun scripts/brand-pack-plan.ts
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Blockers for live brand publish
|
|
29
|
+
|
|
30
|
+
- npm automation token / 2FA for `@sylphx` scope
|
|
31
|
+
- Changeset/version alignment with transitional package
|
|
32
|
+
- Registry readback proof after publish
|
|
33
|
+
|
|
34
|
+
## Authority
|
|
35
|
+
|
|
36
|
+
Publish and brand packages only from **this repository**. No central Instruments monorepo.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Lookout — competitive positioning
|
|
2
|
+
|
|
3
|
+
## Job
|
|
4
|
+
|
|
5
|
+
Local-first web search/fetch/extract/cache
|
|
6
|
+
|
|
7
|
+
## Wedge
|
|
8
|
+
|
|
9
|
+
Tiny default: public search adapters + fetch/extract/cache with citeable excerpts; no required API key and no multi-GB warmup.
|
|
10
|
+
|
|
11
|
+
## Local-first
|
|
12
|
+
|
|
13
|
+
Default public adapters; no required API key; SSRF-safe fetch.
|
|
14
|
+
|
|
15
|
+
## Peer anchors (learn; do not clone)
|
|
16
|
+
|
|
17
|
+
| Peer | Gap we exploit |
|
|
18
|
+
| --- | --- |
|
|
19
|
+
| KnockOutEZ/wigolo | Local web stack for agents; often heavier browser/model warmup paths |
|
|
20
|
+
| Paid search API MCPs | API keys, quota, non-local |
|
|
21
|
+
| Raw fetch without extract/cache | No citeable excerpts or repeatable cache |
|
|
22
|
+
|
|
23
|
+
## Non-goals
|
|
24
|
+
|
|
25
|
+
- Becoming a cloud SaaS wrapper as the default path
|
|
26
|
+
- Multi-product monorepo for star aggregation
|
|
27
|
+
- Generative summaries as the sole evidence authority
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Evidence contract — product-specific
|
|
2
|
+
|
|
3
|
+
**Evidence First** means results carry citeable structure. There is **no** MCP tool named `evidence_first`.
|
|
4
|
+
|
|
5
|
+
## Locators and honesty for this product
|
|
6
|
+
|
|
7
|
+
- result URL + rank
|
|
8
|
+
- excerpt spans
|
|
9
|
+
- fetch route + cache hit/miss
|
|
10
|
+
- adapter warnings (no paid key required on default path)
|
|
11
|
+
|
|
12
|
+
## Always include when applicable
|
|
13
|
+
|
|
14
|
+
- **route**: which local engine path produced the payload
|
|
15
|
+
- **warnings**: missing binaries, partial parse, network/adapter limits
|
|
16
|
+
- raw facts over generative rewrite as authority
|
|
17
|
+
|
|
18
|
+
## Non-goals
|
|
19
|
+
|
|
20
|
+
- Requiring a cloud model to “confirm” local facts
|
|
21
|
+
- Over-marketing Evidence First without locators on the wire
|
package/docs/IPPB.md
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Independent Public Product Bar (IPPB)
|
|
2
|
+
|
|
3
|
+
This repository is complete for the company programme only when all are true:
|
|
4
|
+
|
|
5
|
+
1. **Independence** — product SSOT here; no central Instruments monorepo/hub authority
|
|
6
|
+
2. **Surfaces** — SDK + CLI + MCP documented and present
|
|
7
|
+
3. **Tools** — few clear tools; schema not agent-confusing
|
|
8
|
+
4. **Evidence** — result contract with locators/routes/warnings (not `evidence_first` tool)
|
|
9
|
+
5. **Local-first** — default path needs no cloud API key
|
|
10
|
+
6. **Validation** — release-gate or declared test path green on tip
|
|
11
|
+
7. **Public marketing** — README explains why-us, 30s install, competitive wedge
|
|
12
|
+
8. **Competitive** — `docs/COMPETITIVE.md` anchors peers honestly
|
|
13
|
+
9. **Packaging** — npm package published **or** explicit external publish blocker + git install path
|
|
14
|
+
10. **Clean** — no archived family package deps; no misleading family SSOT docs
|
|
15
|
+
|
|
16
|
+
Stars (e.g. 10k) are growth outcomes, not substitutes for the bar above.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Lookout — positioning
|
|
2
|
+
|
|
3
|
+
## One-liner
|
|
4
|
+
|
|
5
|
+
**Lookout**: Local-first web search/fetch/extract/cache — local-first, fast, light, powerful.
|
|
6
|
+
|
|
7
|
+
## Why agents use this
|
|
8
|
+
|
|
9
|
+
Tiny default: public search adapters + fetch/extract/cache with citeable excerpts; no required API key and no multi-GB warmup.
|
|
10
|
+
|
|
11
|
+
## Surfaces
|
|
12
|
+
|
|
13
|
+
| Surface | Role |
|
|
14
|
+
| --- | --- |
|
|
15
|
+
| MCP | Agent tools over stdio |
|
|
16
|
+
| CLI | Human/scriptable brand bin |
|
|
17
|
+
| SDK | Programmatic library for apps and internal dogfood |
|
|
18
|
+
|
|
19
|
+
## Primary tools
|
|
20
|
+
|
|
21
|
+
- `web_search`
|
|
22
|
+
- `web_fetch`
|
|
23
|
+
- `web_extract`
|
|
24
|
+
- `web_cache`
|
|
25
|
+
- `web_crawl (advanced)`
|
|
26
|
+
- `web_research (advanced)`
|
|
27
|
+
|
|
28
|
+
## Evidence
|
|
29
|
+
|
|
30
|
+
See [EVIDENCE_CONTRACT.md](./EVIDENCE_CONTRACT.md).
|
|
31
|
+
|
|
32
|
+
## Independence
|
|
33
|
+
|
|
34
|
+
See [PRODUCT_INDEPENDENCE.md](./PRODUCT_INDEPENDENCE.md).
|
|
35
|
+
|
|
36
|
+
## Competitive
|
|
37
|
+
|
|
38
|
+
See [COMPETITIVE.md](./COMPETITIVE.md).
|
|
39
|
+
|
|
40
|
+
## Completion bar
|
|
41
|
+
|
|
42
|
+
See [IPPB.md](./IPPB.md).
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Product independence
|
|
2
|
+
|
|
3
|
+
**Lookout** (`lookout`) is an **independent product**.
|
|
4
|
+
|
|
5
|
+
This repository is the only source of truth for marketplace listing, release train,
|
|
6
|
+
package/CLI/MCP/SDK contracts, docs, tests, and stars.
|
|
7
|
+
|
|
8
|
+
- No multi-product monorepo shipping
|
|
9
|
+
- No `SylphxAI/instruments` authority (retired)
|
|
10
|
+
- No archived `@sylphx/reader-evidence` control package
|
|
11
|
+
- Composition with siblings is via **public contracts** only
|
|
12
|
+
|
|
13
|
+
Company private thin map (internal): `SylphxAI/portfolio`.
|
package/docs/PUBLISH.md
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Publish status — Lookout
|
|
2
|
+
|
|
3
|
+
| Field | Value |
|
|
4
|
+
| --- | --- |
|
|
5
|
+
| Package | `@sylphx/lookout` |
|
|
6
|
+
| Repo version | `0.1.0` |
|
|
7
|
+
| Registry state | **not on npm (404)** |
|
|
8
|
+
| npm auth in this environment | `ENEEDAUTH` (external credential blocker) |
|
|
9
|
+
|
|
10
|
+
## In-repo readiness
|
|
11
|
+
|
|
12
|
+
- CI/tests/release-gate green on tip
|
|
13
|
+
- `npm pack --dry-run` produces a valid TS package (~23KB)
|
|
14
|
+
- Live `npm publish` blocked only by missing `@sylphx` registry auth in this environment
|
|
15
|
+
|
|
16
|
+
## Install until publish lands
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
git clone https://github.com/SylphxAI/lookout.git
|
|
20
|
+
cd lookout
|
|
21
|
+
bun install
|
|
22
|
+
./bin/lookout doctor
|
|
23
|
+
```
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Tool surface — this product
|
|
2
|
+
|
|
3
|
+
Policy: **few, powerful, obvious** tools. Prefer the primary read tool first.
|
|
4
|
+
|
|
5
|
+
| Tool | Role |
|
|
6
|
+
| --- | --- |
|
|
7
|
+
| `web_search` | Local-first public search adapters |
|
|
8
|
+
| `web_fetch` | SSRF-aware fetch |
|
|
9
|
+
| `web_extract` | Main-content extract with citeable spans |
|
|
10
|
+
| `web_cache` | Cache stats/control |
|
|
11
|
+
| `web_crawl` | Advanced crawl |
|
|
12
|
+
| `web_research` | Advanced multi-step research |
|
|
13
|
+
|
|
14
|
+
## Rules
|
|
15
|
+
|
|
16
|
+
1. Do not add near-duplicate tools that only differ by vanity naming.
|
|
17
|
+
2. Advanced tools must be labeled advanced in README/skill.
|
|
18
|
+
3. Schema fields should be agent-obvious; fail closed on unsafe input.
|
|
19
|
+
4. Composition with sibling products is via public contracts, not monorepo imports.
|
package/package.json
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@sylphx/lookout",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"private": false,
|
|
5
|
+
"description": "Lookout \u2014 local-first web instrument for agents. Search, fetch, extract, cache with citeable excerpts. No required API key. SDK \u00b7 CLI \u00b7 MCP.",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"license": "MIT",
|
|
8
|
+
"bin": {
|
|
9
|
+
"lookout": "./bin/lookout"
|
|
10
|
+
},
|
|
11
|
+
"exports": {
|
|
12
|
+
".": "./src/sdk.ts",
|
|
13
|
+
"./sdk": "./src/sdk.ts",
|
|
14
|
+
"./engine": "./src/engine.ts"
|
|
15
|
+
},
|
|
16
|
+
"files": [
|
|
17
|
+
"LICENSE",
|
|
18
|
+
"README.md",
|
|
19
|
+
"bin",
|
|
20
|
+
"docs",
|
|
21
|
+
"server.json",
|
|
22
|
+
"src"
|
|
23
|
+
],
|
|
24
|
+
"scripts": {
|
|
25
|
+
"test": "bun test",
|
|
26
|
+
"start": "bun src/cli.ts",
|
|
27
|
+
"mcp": "bun src/mcp.ts",
|
|
28
|
+
"doctor": "./bin/lookout doctor",
|
|
29
|
+
"benchmark:release-gate": "bun scripts/release-gate.ts",
|
|
30
|
+
"validate": "bun test && bun scripts/release-gate.ts",
|
|
31
|
+
"benchmark:public-proof": "bun scripts/public-proof.ts",
|
|
32
|
+
"brand:pack-plan": "bun scripts/brand-pack-plan.ts"
|
|
33
|
+
},
|
|
34
|
+
"dependencies": {
|
|
35
|
+
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
36
|
+
"zod": "^3.25.76"
|
|
37
|
+
},
|
|
38
|
+
"engines": {
|
|
39
|
+
"node": ">=20"
|
|
40
|
+
},
|
|
41
|
+
"repository": {
|
|
42
|
+
"type": "git",
|
|
43
|
+
"url": "git+https://github.com/SylphxAI/lookout.git"
|
|
44
|
+
},
|
|
45
|
+
"homepage": "https://github.com/SylphxAI/lookout#readme",
|
|
46
|
+
"mcpName": "io.github.SylphxAI/lookout",
|
|
47
|
+
"keywords": [
|
|
48
|
+
"mcp",
|
|
49
|
+
"local-first",
|
|
50
|
+
"web-search",
|
|
51
|
+
"lookout",
|
|
52
|
+
"sylphx",
|
|
53
|
+
"agent"
|
|
54
|
+
],
|
|
55
|
+
"publishConfig": {
|
|
56
|
+
"access": "public"
|
|
57
|
+
}
|
|
58
|
+
}
|
package/server.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
|
|
3
|
+
"name": "io.github.SylphxAI/lookout",
|
|
4
|
+
"title": "Lookout",
|
|
5
|
+
"description": "Local-first web instrument for agents: search, fetch, extract, cache. No required API key. SDK · CLI · MCP.",
|
|
6
|
+
"repository": {
|
|
7
|
+
"url": "https://github.com/SylphxAI/lookout",
|
|
8
|
+
"source": "github"
|
|
9
|
+
},
|
|
10
|
+
"version": "0.1.0",
|
|
11
|
+
"websiteUrl": "https://github.com/SylphxAI/lookout#readme",
|
|
12
|
+
"packages": [
|
|
13
|
+
{
|
|
14
|
+
"registryType": "npm",
|
|
15
|
+
"identifier": "@sylphx/lookout",
|
|
16
|
+
"version": "0.1.0",
|
|
17
|
+
"transport": {
|
|
18
|
+
"type": "stdio"
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
]
|
|
22
|
+
}
|
package/src/cache.ts
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import { homedir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
|
|
6
|
+
export type CacheRecord = {
|
|
7
|
+
key: string;
|
|
8
|
+
kind: string;
|
|
9
|
+
createdAt: string;
|
|
10
|
+
url?: string;
|
|
11
|
+
query?: string;
|
|
12
|
+
contentType?: string;
|
|
13
|
+
body: string;
|
|
14
|
+
meta?: Record<string, unknown>;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
export function defaultCacheDir(): string {
|
|
18
|
+
if (process.env.LOOKOUT_CACHE_DIR?.trim()) return process.env.LOOKOUT_CACHE_DIR.trim();
|
|
19
|
+
const xdg = process.env.XDG_CACHE_HOME?.trim();
|
|
20
|
+
if (xdg) return join(xdg, 'lookout');
|
|
21
|
+
return join(homedir(), '.cache', 'lookout');
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export class LookoutCache {
|
|
25
|
+
readonly dir: string;
|
|
26
|
+
|
|
27
|
+
constructor(dir = defaultCacheDir()) {
|
|
28
|
+
this.dir = dir;
|
|
29
|
+
mkdirSync(this.dir, { recursive: true });
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
private pathFor(key: string): string {
|
|
33
|
+
const safe = createHash('sha256').update(key).digest('hex');
|
|
34
|
+
return join(this.dir, `${safe}.json`);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
put(record: Omit<CacheRecord, 'createdAt'> & { createdAt?: string }): CacheRecord {
|
|
38
|
+
const full: CacheRecord = {
|
|
39
|
+
...record,
|
|
40
|
+
createdAt: record.createdAt ?? new Date().toISOString(),
|
|
41
|
+
};
|
|
42
|
+
writeFileSync(this.pathFor(full.key), JSON.stringify(full), 'utf8');
|
|
43
|
+
return full;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
get(key: string, options: { maxAgeMs?: number } = {}): CacheRecord | null {
|
|
47
|
+
const p = this.pathFor(key);
|
|
48
|
+
if (!existsSync(p)) return null;
|
|
49
|
+
try {
|
|
50
|
+
const rec = JSON.parse(readFileSync(p, 'utf8')) as CacheRecord;
|
|
51
|
+
if (typeof options.maxAgeMs === 'number' && options.maxAgeMs >= 0) {
|
|
52
|
+
const created = Date.parse(rec.createdAt);
|
|
53
|
+
if (!Number.isFinite(created) || Date.now() - created > options.maxAgeMs) {
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return rec;
|
|
58
|
+
} catch {
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
query(q: string, limit = 20): CacheRecord[] {
|
|
64
|
+
const needle = q.toLowerCase();
|
|
65
|
+
const out: CacheRecord[] = [];
|
|
66
|
+
for (const name of readdirSync(this.dir)) {
|
|
67
|
+
if (!name.endsWith('.json')) continue;
|
|
68
|
+
try {
|
|
69
|
+
const rec = JSON.parse(readFileSync(join(this.dir, name), 'utf8')) as CacheRecord;
|
|
70
|
+
const hay = `${rec.url ?? ''} ${rec.query ?? ''} ${rec.body.slice(0, 2000)}`.toLowerCase();
|
|
71
|
+
if (!needle || hay.includes(needle)) out.push(rec);
|
|
72
|
+
} catch {
|
|
73
|
+
// skip corrupt
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
out.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
|
|
77
|
+
return out.slice(0, limit);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
stats(): { entries: number; dir: string; bytes: number } {
|
|
81
|
+
let entries = 0;
|
|
82
|
+
let bytes = 0;
|
|
83
|
+
for (const name of readdirSync(this.dir)) {
|
|
84
|
+
if (!name.endsWith('.json')) continue;
|
|
85
|
+
entries += 1;
|
|
86
|
+
try {
|
|
87
|
+
bytes += statSync(join(this.dir, name)).size;
|
|
88
|
+
} catch {
|
|
89
|
+
// ignore
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return { entries, dir: this.dir, bytes };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
clear(): { removed: number } {
|
|
96
|
+
let removed = 0;
|
|
97
|
+
for (const name of readdirSync(this.dir)) {
|
|
98
|
+
if (!name.endsWith('.json')) continue;
|
|
99
|
+
rmSync(join(this.dir, name), { force: true });
|
|
100
|
+
removed += 1;
|
|
101
|
+
}
|
|
102
|
+
return { removed };
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Remove entries older than maxAgeMs. */
|
|
106
|
+
pruneExpired(maxAgeMs: number): { removed: number } {
|
|
107
|
+
let removed = 0;
|
|
108
|
+
const now = Date.now();
|
|
109
|
+
for (const name of readdirSync(this.dir)) {
|
|
110
|
+
if (!name.endsWith('.json')) continue;
|
|
111
|
+
try {
|
|
112
|
+
const rec = JSON.parse(readFileSync(join(this.dir, name), 'utf8')) as CacheRecord;
|
|
113
|
+
const created = Date.parse(rec.createdAt);
|
|
114
|
+
if (!Number.isFinite(created) || now - created > maxAgeMs) {
|
|
115
|
+
rmSync(join(this.dir, name), { force: true });
|
|
116
|
+
removed += 1;
|
|
117
|
+
}
|
|
118
|
+
} catch {
|
|
119
|
+
rmSync(join(this.dir, name), { force: true });
|
|
120
|
+
removed += 1;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
return { removed };
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export function cacheKey(parts: string[]): string {
|
|
128
|
+
return parts.join('|');
|
|
129
|
+
}
|
package/src/cli.ts
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
import { LookoutEngine, CORE_TOOLS, ADVANCED_TOOLS } from './engine.ts';
|
|
3
|
+
|
|
4
|
+
function usage(): never {
|
|
5
|
+
console.log(`Lookout — local-first web instrument (Sylphx)
|
|
6
|
+
|
|
7
|
+
Usage:
|
|
8
|
+
lookout search <query...>
|
|
9
|
+
lookout fetch <url>
|
|
10
|
+
lookout extract <url>
|
|
11
|
+
lookout cache [query]
|
|
12
|
+
lookout cache stats
|
|
13
|
+
lookout cache clear
|
|
14
|
+
lookout cache prune [maxAgeMs]
|
|
15
|
+
lookout crawl <url> [--depth N] [--pages N]
|
|
16
|
+
lookout research <query...> [--pages N]
|
|
17
|
+
lookout mcp
|
|
18
|
+
lookout tools
|
|
19
|
+
lookout doctor
|
|
20
|
+
lookout help
|
|
21
|
+
|
|
22
|
+
Env:
|
|
23
|
+
LOOKOUT_CACHE_DIR override cache directory (default: ~/.cache/lookout)
|
|
24
|
+
LOOKOUT_CACHE_MAX_AGE_MS optional max age for cache hits (milliseconds)
|
|
25
|
+
`);
|
|
26
|
+
process.exit(0);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
async function main() {
|
|
30
|
+
const [cmd, ...rest] = process.argv.slice(2);
|
|
31
|
+
if (!cmd || cmd === 'help' || cmd === '-h' || cmd === '--help') usage();
|
|
32
|
+
|
|
33
|
+
if (cmd === 'tools') {
|
|
34
|
+
console.log(JSON.stringify({ core: CORE_TOOLS, advanced: ADVANCED_TOOLS }, null, 2));
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
if (cmd === 'doctor') {
|
|
39
|
+
const { runDoctor, formatDoctorReport } = await import('./doctor.ts');
|
|
40
|
+
const report = runDoctor();
|
|
41
|
+
console.log(formatDoctorReport(report));
|
|
42
|
+
process.exit(report.ok ? 0 : 1);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
if (cmd === 'mcp') {
|
|
46
|
+
await import('./mcp.ts');
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const engine = new LookoutEngine();
|
|
51
|
+
let envelope;
|
|
52
|
+
switch (cmd) {
|
|
53
|
+
case 'search':
|
|
54
|
+
envelope = await engine.handle('web_search', { query: rest.join(' ') });
|
|
55
|
+
break;
|
|
56
|
+
case 'fetch':
|
|
57
|
+
envelope = await engine.handle('web_fetch', { url: rest[0] });
|
|
58
|
+
break;
|
|
59
|
+
case 'extract':
|
|
60
|
+
envelope = await engine.handle('web_extract', { url: rest[0] });
|
|
61
|
+
break;
|
|
62
|
+
case 'cache':
|
|
63
|
+
if (rest[0] === 'stats') envelope = await engine.handle('web_cache', { op: 'stats' });
|
|
64
|
+
else if (rest[0] === 'clear') envelope = await engine.handle('web_cache', { op: 'clear' });
|
|
65
|
+
else if (rest[0] === 'prune') {
|
|
66
|
+
const maxAgeMs = rest[1] ? Number(rest[1]) : undefined;
|
|
67
|
+
envelope = await engine.handle('web_cache', { op: 'prune', maxAgeMs });
|
|
68
|
+
} else envelope = await engine.handle('web_cache', { op: 'query', query: rest.join(' ') });
|
|
69
|
+
break;
|
|
70
|
+
case 'crawl': {
|
|
71
|
+
let depth: number | undefined;
|
|
72
|
+
let pages: number | undefined;
|
|
73
|
+
const args = [...rest];
|
|
74
|
+
const url = args.shift();
|
|
75
|
+
while (args.length) {
|
|
76
|
+
const a = args.shift()!;
|
|
77
|
+
if (a === '--depth') depth = Number(args.shift());
|
|
78
|
+
else if (a === '--pages') pages = Number(args.shift());
|
|
79
|
+
}
|
|
80
|
+
envelope = await engine.handle('web_crawl', { url, maxDepth: depth, maxPages: pages });
|
|
81
|
+
break;
|
|
82
|
+
}
|
|
83
|
+
case 'research': {
|
|
84
|
+
let pages: number | undefined;
|
|
85
|
+
const args = [...rest];
|
|
86
|
+
const qparts: string[] = [];
|
|
87
|
+
while (args.length) {
|
|
88
|
+
const a = args.shift()!;
|
|
89
|
+
if (a === '--pages') pages = Number(args.shift());
|
|
90
|
+
else qparts.push(a);
|
|
91
|
+
}
|
|
92
|
+
envelope = await engine.handle('web_research', { query: qparts.join(' '), maxPages: pages });
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
default:
|
|
96
|
+
console.error(`Unknown command: ${cmd}`);
|
|
97
|
+
usage();
|
|
98
|
+
}
|
|
99
|
+
console.log(JSON.stringify(envelope, null, 2));
|
|
100
|
+
process.exit(envelope?.status === 'ok' ? 0 : 1);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
main().catch((err) => {
|
|
104
|
+
console.error(err);
|
|
105
|
+
process.exit(1);
|
|
106
|
+
});
|