@retiform/paperless-mcp 2.3.5 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +301 -4
  2. package/build/api/PaperlessAPI.js +1 -6
  3. package/build/search/build-outcome.d.ts +28 -0
  4. package/build/search/build-outcome.js +31 -0
  5. package/build/search/chunk.d.ts +16 -0
  6. package/build/search/chunk.js +64 -0
  7. package/build/search/config.d.ts +120 -0
  8. package/build/search/config.js +264 -0
  9. package/build/search/embedder.d.ts +346 -0
  10. package/build/search/embedder.js +734 -0
  11. package/build/search/rank.d.ts +11 -0
  12. package/build/search/rank.js +87 -0
  13. package/build/search/store.d.ts +406 -0
  14. package/build/search/store.js +847 -0
  15. package/build/search/sync.d.ts +204 -0
  16. package/build/search/sync.js +819 -0
  17. package/build/search/webhook.d.ts +193 -0
  18. package/build/search/webhook.js +445 -0
  19. package/build/server.d.ts +29 -0
  20. package/build/server.js +77 -0
  21. package/build/tools/documents.d.ts +13 -3
  22. package/build/tools/documents.js +23 -6
  23. package/build/tools/search.d.ts +518 -0
  24. package/build/tools/search.js +1199 -0
  25. package/build/tools/utils/middlewares.d.ts +5 -0
  26. package/build/tools/utils/middlewares.js +282 -9
  27. package/package.json +10 -3
  28. package/paperless-mcp.dxt +0 -0
  29. package/build/api/documentEnhancer.test.d.ts +0 -1
  30. package/build/api/documentEnhancer.test.js +0 -80
  31. package/build/resources/documents.test.d.ts +0 -1
  32. package/build/resources/documents.test.js +0 -124
  33. package/build/server.test.d.ts +0 -1
  34. package/build/server.test.js +0 -54
  35. package/build/tools/boundedContent.test.d.ts +0 -1
  36. package/build/tools/boundedContent.test.js +0 -565
  37. package/build/tools/documents.test.d.ts +0 -1
  38. package/build/tools/documents.test.js +0 -658
  39. package/build/tools/mail.test.d.ts +0 -1
  40. package/build/tools/mail.test.js +0 -212
  41. package/build/tools/notes.test.d.ts +0 -1
  42. package/build/tools/notes.test.js +0 -177
  43. package/build/tools/schemaCompat.test.d.ts +0 -1
  44. package/build/tools/schemaCompat.test.js +0 -59
  45. package/build/tools/utils/monetary.test.d.ts +0 -1
  46. package/build/tools/utils/monetary.test.js +0 -25
  47. package/build/tools/utils/resourceUri.test.d.ts +0 -1
  48. package/build/tools/utils/resourceUri.test.js +0 -50
  49. package/build/tools/utils/selectFields.test.d.ts +0 -1
  50. package/build/tools/utils/selectFields.test.js +0 -164
package/README.md CHANGED
@@ -15,7 +15,8 @@ This is a fork of [baruchiro/paperless-mcp](https://github.com/baruchiro/paperle
15
15
  Upstream returns whole documents. That is fine for a human clicking through a UI
16
16
  and crippling for an agent: a single `get_document_content` call on a
17
17
  several-hundred-page PDF can consume an entire context window, and there is no
18
- way to ask for a slice. This fork adds bounded reads:
18
+ way to ask for a slice. This fork adds bounded reads and local semantic
19
+ search:
19
20
 
20
21
  - **Ranged `get_document_content`** -- `offset` + `max_chars` windows, with honest
21
22
  `content_length` / `truncated` / `has_more` metadata.
@@ -28,6 +29,11 @@ way to ask for a slice. This fork adds bounded reads:
28
29
  before making it.
29
30
  - **Query resilience** -- an apostrophe in a search query is retried sanitized
30
31
  instead of returning a hard 400.
32
+ - **Semantic search** -- `semantic_search` searches by meaning as well as by
33
+ words, fusing a local vector index with the lexical search above and
34
+ returning the same bounded-passage shape as `search_passages`. Optional,
35
+ local-only, and built explicitly with `sync_embeddings`; `embedding_status`
36
+ reports what the index holds. See [Semantic search setup](#semantic-search-setup).
31
37
 
32
38
  We also fix the build so it is idempotent (`tsc` no longer consumes its own
33
39
  `.d.ts` output on a rebuild). Upstream was unable to take this work, and agentic
@@ -96,7 +102,27 @@ Add these to your MCP config file:
96
102
  | `PAPERLESS_PUBLIC_URL` | No | `PAPERLESS_URL` | Public-facing URL for document links |
97
103
  | `PAPERLESS_API_VERSION` | No | `9` | Paperless-ngx REST API version. `9` works on Paperless-ngx v2.x (recent) and v3.x. Paperless-ngx v3.0.0 dropped support for versions below `9`, so older defaults now return HTTP 406. If you see HTTP 406 errors, set this to a version your server supports. |
98
104
  | `PAPERLESS_MCP_UPLOAD_PATHS` | No | — | Colon-separated list of allowed directories for `file_path` uploads. **Recommended for security.** Example: `/var/uploads:/tmp/scans` |
99
- | `PAPERLESS_MCP_MAX_INLINE_CHARS` | No | `50000` | Hard cap on characters a single tool response may inline. Requests above the budget fail with the document size and a hint toward bounded reads instead of silently truncating or flooding context. |
105
+ | `PAPERLESS_MCP_MAX_INLINE_CHARS` | No | `20000` | Hard cap on characters a single tool response may inline. Requests above the budget fail with the document size and a hint toward bounded reads instead of silently truncating or flooding context. The default matches the 20,000-char ceiling most MCP clients apply to a tool result; setting it higher means the response is built whole and then truncated downstream without your caller being able to tell. |
106
+ | `PAPERLESS_MCP_EMBEDDING_URL` *(Search)* | No | `http://127.0.0.1:11434` | Base URL of an OpenAI-compatible `/embeddings` endpoint. Either reply envelope is accepted — `{embeddings: [[...]]}` (Ollama native) or `{data: [{embedding: [...]}]}` (OpenAI-compatible) — so no setting tells the client which shape to expect. Setting this (or any other `PAPERLESS_MCP_*` search variable) enables the semantic-search tools. |
107
+ | `PAPERLESS_MCP_EMBEDDING_LOCAL_ONLY` *(Search)* | No | `false` | When `true`, the endpoint must be a literal loopback (`127.0.0.1`, `::1`) or RFC1918 address (`10.x`, `172.16-31.x`, `192.168.x`); a DNS name is refused, because a name can resolve anywhere. Set this if indexed text must not leave your network. Off by default so a hosted provider works without a policy you did not ask for. |
108
+ | `PAPERLESS_MCP_EMBEDDING_API_KEY` *(Search)* | No | — | Sent as `Authorization: Bearer <key>`. Never logged, never included in an error message, and held in a private field so it does not appear in a serialised client object. Leaving it unset sends no authorization header at all. |
109
+ | `PAPERLESS_MCP_EMBEDDING_MODEL` *(Search)* | No | `nomic-embed-text` | Embedding model name, requested from the server above. Must produce 768-dimension vectors. |
110
+ | `PAPERLESS_MCP_EMBEDDING_PREFIX_MODE` *(Search)* | No | `nomic` | One of `nomic`, `none`, `e5` — the prefix template applied to embedded text. The prefix is **baked into every stored vector at build time**, so a query built with a different mode returns quietly worse results rather than an error. A mismatch between the stored mode and the configured one is refused loudly; changing this requires a rebuild. |
111
+ | `PAPERLESS_MCP_INDEX_DIR` *(Search)* | No | `~/.local/share/paperless-mcp/index` | Directory holding `index.db`. Outside your Paperless-ngx container by design. |
112
+ | `PAPERLESS_MCP_MAX_CHUNKS` *(Search)* | No | `4000` | Maximum chunks stored per document. A longer document is truncated, and `embedding_status` reports how many were. Clamped to what the row-id encoding can address; a larger value is reduced rather than refused. |
113
+ | `PAPERLESS_MCP_SYNC_BATCH_SIZE` *(Search)* | No | `16` | Texts sent per embedding request during a sync (1–256). Lower it if your embedding server falls over on larger batches. |
114
+ | `PAPERLESS_MCP_SYNC_INTERVAL_MS` *(Search)* | No | `0` (off) | Interval between automatic incremental syncs (0–86400000). A positive value starts a background timer when the tools register; it runs one sync immediately, then once per interval. `0` means no timer, which is the default — drive `sync_embeddings` from your client instead. The timer is cleared on `close()`. |
115
+ | `PAPERLESS_MCP_WEBHOOK_PORT` *(Search)* | No | `0` (off) | Port for the change-notification receiver. A positive value starts a listener that re-indexes documents as they change, so the index stays current without anything polling. **Off by default**: an install that never asked for a listening socket must not acquire one by upgrading. Requires `PAPERLESS_MCP_WEBHOOK_SECRET`; the receiver refuses to start without one. |
116
+ | `PAPERLESS_MCP_WEBHOOK_SECRET` *(Search)* | Only with the port | — | Shared secret the sender must present, in the `x-paperless-mcp-secret` header or as a `Bearer` token. Compared in constant time. The receiver **refuses to start** when this is unset and the port is set, because an unauthenticated endpoint would let anyone trigger embedding work. |
117
+ | `PAPERLESS_MCP_WEBHOOK_HOST` *(Search)* | No | `127.0.0.1` | Address to bind. Loopback unless you widen it deliberately. **A containerised Paperless-ngx cannot reach `127.0.0.1` on the host** — see the receiver section below. |
118
+ | `PAPERLESS_MCP_WEBHOOK_DEBOUNCE_MS` *(Search)* | No | `2000` | Quiet period before a burst of deliveries becomes one sync. A bulk edit arrives as one request per document within a few seconds; coalescing them means 200 events cost one run, not 200. |
119
+
120
+ Rows marked *(Search)* belong to the optional semantic-search feature: **setting any one of them
121
+ enables the three search tools**, and each value you set is validated strictly rather than silently
122
+ defaulted. Four of them also accept an unprefixed fallback name — `EMBEDDING_URL`, `EMBEDDING_MODEL`,
123
+ `EMBEDDING_PREFIX_MODE`, `SYNC_INTERVAL_MS` — tried only when the prefixed form is unset. Prefer the
124
+ prefixed names: the unprefixed ones are generic, so any other MCP server sharing the same process
125
+ environment can collide with them.
100
126
 
101
127
  That's it! Now you can ask Claude to help you manage your Paperless-NGX documents.
102
128
 
@@ -245,10 +271,12 @@ get_document_content({ id: 123, offset: 8000, max_chars: 8000 })
245
271
  // → next window
246
272
  ```
247
273
 
248
- Requests above the inline budget (`PAPERLESS_MCP_MAX_INLINE_CHARS`, default 50000) fail with the document size and a hint toward offset reads or the `download_document` resource — never a silent slice, never an unbounded return. The response always carries `content_length`, `truncated`, and `has_more` so callers can page deterministically.
274
+ Requests above the inline budget (`PAPERLESS_MCP_MAX_INLINE_CHARS`, default 20000) fail with the document size and a hint toward offset reads or the `download_document` resource — never a silent slice, never an unbounded return. The response always carries `content_length`, `truncated`, and `has_more` so callers can page deterministically.
249
275
 
250
276
  #### search_passages
251
- Search documents full-text and return **bounded text passages around each hit** instead of whole documents. Each window is anchored on the server's own highlighted phrase (falling back to the first verbatim term occurrence, then the document head), so the returned passage contains the actual hit even when the search index stemmed a word variant that does not appear verbatim in the stored text. Returns the document id, title, score, offset, `content_length`, and the highlighted snippet.
277
+ Search documents full-text and return **bounded text passages around each hit** instead of whole documents. Each window is anchored on the server's own highlighted phrase (falling back to the **longest matching query term**, then the document head), so the returned passage contains the actual hit even when the search index stemmed a word variant that does not appear verbatim in the stored text. Returns the document id, title, score, offset, `content_length`, and the highlighted snippet.
278
+
279
+ ⚠️ **This tool and `semantic_search` rank their fallback anchor differently, on purpose.** `semantic_search` orders candidates by corpus **rarity**, because it holds the index. This tool has no index access and never will, so it uses term **length** as its proxy. When the server's highlight resolves — which is the common case here — neither fallback runs and the two agree exactly.
252
280
 
253
281
  Queries that the server rejects with an opaque HTTP 400 (django-watson rejects some syntactically valid queries, notably those containing apostrophes) are retried once with a defensively rewritten query — apostrophes and stray quotes removed; structured filter values are never rewritten. Responses from a rewritten retry carry `query_sanitized: true`. Persistent failures throw a structured error naming every attempted query.
254
282
 
@@ -784,6 +812,275 @@ delete_mail_rule({
784
812
  })
785
813
  ```
786
814
 
815
+ ### Semantic Search
816
+
817
+ Three optional tools that search the corpus by **meaning** as well as by words, using a vector index fused with the corpus' own lexical (BM25) search. They register only when at least one of the search variables below is set — the nine marked *Search* in the Environment Variables table, not `PAPERLESS_MCP_MAX_INLINE_CHARS` or `PAPERLESS_MCP_UPLOAD_PATHS`. With none of the nine set, the server behaves exactly as before.
818
+
819
+ Any OpenAI-compatible `/embeddings` endpoint works: a local server such as Ollama, or a hosted provider with an API key. Both reply envelopes are accepted without configuration. If indexed text must not leave your network, set `PAPERLESS_MCP_EMBEDDING_LOCAL_ONLY=true` — the endpoint is then required to be a literal loopback or RFC 1918 address and a DNS name is refused. It is off by default so an install pointing at a hosted provider is not fighting a policy it never chose.
820
+
821
+ #### semantic_search
822
+ Search by meaning and by words together, returning bounded passages in the same shape as `search_passages`.
823
+
824
+ Parameters:
825
+ - query: What to search for, in natural language
826
+ - limit (optional): Maximum passages to return (default 10, max 50)
827
+ - window (optional): Characters per passage (default 800, range 200–4000)
828
+ - correspondent, document_type, tag, storage_path, created__date__gte, created__date__lte (optional): Same filters as `search_passages`, and every one of them is honoured. A document with no filing location cannot match a numeric `storage_path` — asking for a storage path means asking for a document that has one.
829
+
830
+ ```typescript
831
+ semantic_search({ query: "what did the inspector say about the roof" })
832
+ // → {
833
+ // query, semantic: "ok", strong_match: true, total_hits: 4,
834
+ // passages: [{ id, title, score, matched, highlights, passage, offset, content_length, offset_space }, ...]
835
+ // }
836
+ ```
837
+
838
+ **Return shape.** `passages[]` mirrors `search_passages` — `id`, `title`, `score`, `passage`, `offset`, `content_length` — so a client that has learned one has learned the other. `matched` is added, saying whether the hit came from meaning, words, or both.
839
+
840
+ ⚠️ **`score` is a reciprocal-rank-fusion sum, not a similarity.** Do not read it against a 0–1 scale, and do not threshold it. For each list a document appears in it contributes `1 / (k + rank)` using its best rank there, so with two retrievers the maximum is `2 / (k + 1)` where `k` is the fusion constant:
841
+
842
+ ```
843
+ score = Σ_arms 1 / (k + bestRank_in_that_arm)
844
+ ```
845
+
846
+ **The ceiling is a function of `k` and moves when `k` moves** — it is `2/11 ≈ 0.182` at the shipped `k = 10`. A score near the top means "both retrievers ranked this document first", which is the strongest agreement this scale can express; a score of `0.05` is not "5% relevant", it is a lower position in the same sum. Rank order is the meaning; the magnitude is not portable between versions. If you need a relevance judgement, use `matched` (which halves found it) and the passages themselves.
847
+
848
+ **Where the window anchors.** On the **rarest matching query term**, measured as that term's chunk count across the index and computed once per request. Length is only the tiebreak. It went through two earlier heuristics, and both were wrong in instructive ways: it first anchored on the first term in query order, and English queries open with articles — `"the practice of a wrathful deity with a lion head"` anchored on `"the"` at character 3 and the window landed at the document head. Anchoring on the *longest* term fixed that case and then failed the opposite one: `"doctor"` is longer than `"ankle"` but far commoner (measured 1,903 chunks against 808), so a query containing both anchored on the less distinctive word. Rarity matched a human read 3/3 where length managed 0/3. If no query term appears in the document at all, the window falls back to offset 0 and reports no anchoring term; that is a deliberate, honest head window rather than a fabricated match.
849
+
850
+ ⚠️ **`offset` is in the RECONSTRUCTED text space, not the raw file.** The passage is rebuilt by joining the document's chunks, so chunk *p* begins at `(p−1)×1800`. `offset` and `content_length` address that reconstruction. **Each passage carries `offset_space`** — `"normalized"` on this path, `"document"` when it fell back to the API's own content — so a caller can tell which space it is in without assuming. Do not feed these offsets to a raw-file byte reader.
851
+
852
+ **`matched`** is `"meaning"`, `"words"`, or `"both"`. It tells you whether to trust the hit or re-query lexically: `"words"` means the hit came from the lexical side and the vector side did not find it, so **do not read a words-only result as absence of a paraphrase** that is worded differently.
853
+
854
+ **`total_hits`** is the number of distinct documents the ranking returned **before** your filter
855
+ arguments were applied, so it can exceed `passages.length`. The difference is not left unexplained:
856
+
857
+ - **`filtered_out`** — how many candidates your filter arguments dropped. When non-zero, `filtered_out_reasons` says which filter did it: `metadata_filter` (a filter you passed) or `document_unavailable` (the document existed in the ranking but could no longer be fetched).
858
+ - **`retrievers`** — which halves of the search actually ran, as `"vector"` and/or `"lexical"`. An empty `passages` with both names present is a real result: both engines ran and neither found anything. An empty `passages` with only one name is a partial answer, and `semantic` plus `degraded_reason` say why.
859
+
860
+ **`trimmed`** is present **only when the response was too large to return whole**, and it says exactly what was cut rather than leaving the caller to guess:
861
+
862
+ ```typescript
863
+ trimmed?: {
864
+ reason: "result_size"; // the only reason today
865
+ dropped: number; // how many passages were removed
866
+ limit_chars: 20000; // the ceiling that forced it
867
+ query_truncated?: boolean; // true only when the ECHOED query was cut too
868
+ }
869
+ ```
870
+
871
+ ⚠️ **The bound covers the whole serialised result, not just the passages.** A pathologically long `query` is echoed back, so it is also truncated when necessary — that is what `query_truncated` announces. **A trimmed response is a real answer, not a failure**: the passages you got are the highest-ranked ones, and `dropped` tells you how many you did not.
872
+
873
+ ⚠️ **Note the gap this leaves.** Filtering happens after ranking, so a filtered query can return fewer
874
+ passages than `limit` even when more documents matched the filter. `filtered_out` is how you see that
875
+ happened rather than assuming the corpus is thin.
876
+
877
+ **`strong_match`** is `true` exactly when at least one passage was returned. It is a **coarse "did anything come back" flag, not a quality judgement.**
878
+
879
+ ⚠️ **It used to consult a 0.45 similarity floor, and that floor could not fire on this corpus.** Measured: best raw similarities run **0.73–0.79**, so 0.45 sits *below the worst irrelevant query* — the old field reported `true` on every call that produced a hit, regardless of quality. Rather than keep a field that cannot be `false`, it now says what it actually means.
880
+
881
+ Judge result quality from **`matched`** (meaning / words / both), **`retrievers`** (which halves ran), and the passages themselves. Do not write `if (!strong_match) requery()` — that branch is unreachable.
882
+
883
+ **Degradation contract.** The embedding server being down is an expected condition, not an error, so `semantic_search` never fails because of it. It returns the lexical results it can still produce, with `semantic: "unavailable"` and a `degraded_reason` naming the cause:
884
+
885
+ ```typescript
886
+ // → { semantic: "unavailable", degraded_reason: "connect ECONNREFUSED 127.0.0.1:11434",
887
+ // strong_match: true, total_hits: 1, passages: [...] }
888
+ ```
889
+
890
+ **A degraded call never returns a silent empty list.** `passages: []` alone cannot distinguish "nothing matched" from "half the search engine was missing", so the state is carried explicitly. Three outcomes, and `retrievers` is what tells them apart:
891
+
892
+ | `semantic` | `retrievers` | Meaning |
893
+ |---|---|---|
894
+ | `"ok"` | both | Both engines ran. An empty list here **is** a real answer — nothing matched. |
895
+ | `"unavailable"` | `["lexical"]` | The embedder was unreachable; `degraded_reason` names the cause. Results are word-matches only, so a paraphrase will be missed. |
896
+ | absent | `[]` | Neither engine could run. The tool returns an **error** rather than an empty list that would read as "no such document". |
897
+
898
+ ⚠️ `semantic_search` requires the index to have been built first — see `sync_embeddings` and [Semantic search setup](#semantic-search-setup).
899
+
900
+ #### embedding_status
901
+ Report what the index actually holds. No parameters.
902
+
903
+ ```typescript
904
+ embedding_status()
905
+ // → {
906
+ // documents: 1240, chunks: 9678, dimensions: 768,
907
+ // model: "nomic-embed-text", prefix_mode: "nomic",
908
+ // last_sync: "2026-11-15T12:04:11.000Z",
909
+ // pending: 3, store_bytes: 148373504, truncated_documents: 2,
910
+ // index_path: "/home/you/.local/share/paperless-mcp/index/index.db"
911
+ // }
912
+ ```
913
+
914
+ - `documents` / `chunks` — what is indexed now
915
+ - `dimensions`, `model`, `prefix_mode` — what the index was **built** with; a mismatch against the running configuration is refused at query time
916
+ - `last_sync` — the most recent metadata stamp written, or `null`
917
+ - `pending` — remote document count minus indexed count. **`null` means the count could not be read**, which is not the same as `0`
918
+ - `store_bytes` — size of `index.db` on disk, or `null` if the file does not exist yet
919
+ - `truncated_documents` — documents that hit `PAPERLESS_MCP_MAX_CHUNKS` and lost their tail. **On a truncated document "not found" may be a false negative**, which is why this is reported
920
+ - `open_error` — present only when the index could not be opened, and then the counts above are zeros
921
+
922
+ Use this to tell "the index is empty" apart from "nothing matched".
923
+
924
+ #### sync_embeddings
925
+ Bring the index up to date with the corpus. Long-running on a first build.
926
+
927
+ Parameters:
928
+ - force (optional): Re-embed every document even if `modified` is unchanged. Only needed after a chunker or model change; a normal sync is incremental.
929
+
930
+ ```typescript
931
+ sync_embeddings({})
932
+ // → { added: 12, updated: 3, deleted: 0, metadataRefreshed: 4, failed: 0, deletedSkipped: false }
933
+ ```
934
+
935
+ A normal sync fetches every document whose `modified` changed since the last run, re-chunks and re-embeds it, removes documents that no longer exist, and rebuilds metadata (tags, correspondent, document type) for documents whose metadata age exceeds 24 hours without the document itself being edited.
936
+
937
+ - `failed` counts documents that errored individually and were skipped; `embedding_status` will still show them as pending
938
+ - `deletedSkipped: true` means the remote list changed mid-run (row count disagreed with the reported count), so the delete step was skipped rather than risk removing live documents
939
+
940
+ **An embedding outage aborts the run.** It stops rather than writing an index that is missing documents while claiming to be complete. Run `embedding_status` afterwards to confirm.
941
+
942
+ ⚠️ **`force: true` discards the existing vectors before it rebuilds them, and it proves the embedder is alive first.** The probe is a real embedding of a short constant string, not a health GET — a server that answers GET without its model pulled fails on the first real request, so a GET would clear the index and then discover the problem. If the probe fails, the call refuses and **your existing index is untouched**. Only after a successful probe are the document rows deleted, in the same transaction as their lexical rows. An interrupted rebuild still leaves you with no fallback, so confirm with `embedding_status` afterwards.
943
+
944
+ ## Semantic search setup
945
+
946
+ Semantic search is entirely optional and off until you configure it.
947
+
948
+ 1. **Point it at an embedding server.** The default is `http://127.0.0.1:11434` with `nomic-embed-text`, which is Ollama's out-of-the-box shape. Set `PAPERLESS_MCP_EMBEDDING_URL` to any OpenAI-compatible `/embeddings` endpoint — a local server or a hosted provider — and `PAPERLESS_MCP_EMBEDDING_API_KEY` if that provider needs a bearer token. A base URL with no path is resolved to `/api/embed` (Ollama) and any other path is used as given; a URL already ending in `/embeddings` is used verbatim.
949
+ 2. **Set at least one `PAPERLESS_MCP_*` search variable** (see the Environment Variables table). Any one of them enables the tools; every value you set is validated strictly, and an invalid value refuses loudly rather than silently defaulting.
950
+ 3. **Build the index by calling `sync_embeddings`.** Nothing is built at startup — a first build only happens when you ask for it. Set `PAPERLESS_MCP_SYNC_INTERVAL_MS` to a positive value if you want incremental syncs on a timer afterwards; the first run is immediate, and the timer is cleared when the server closes. On a large corpus the first build takes tens of minutes; call `embedding_status` to watch `documents` and `chunks` climb.
951
+ 4. **Search.** `semantic_search` fuses the vector index with the corpus' lexical search.
952
+
953
+ The index lives at `~/.local/share/paperless-mcp/index/index.db` by default, or wherever `PAPERLESS_MCP_INDEX_DIR` points. It is deliberately **outside** your Paperless-ngx container: it is derived data, it is regenerable by re-running `sync_embeddings`, and putting it inside the container would drag it into that container's backup scope.
954
+
955
+ **Keeping text on your own network.** If indexed content must not leave the host, set `PAPERLESS_MCP_EMBEDDING_LOCAL_ONLY=true`. The endpoint is then required to be a literal loopback or RFC 1918 address, and a DNS name is refused — a name can resolve to anywhere. By default this restriction is off, because the package is published for general use and a third-party install must be able to point it at whichever provider it has.
956
+
957
+ ## Keeping the index current without polling
958
+
959
+ An incremental sync compares a recorded watermark against the live corpus, so it can tell you
960
+ "nothing changed" in a single request. That works for additions, edits and deletions — but it is
961
+ **structurally blind to one case**: a tag, correspondent or document-type edit is applied through a
962
+ bulk update that never touches a document's `modified` timestamp, and never changes the document
963
+ count. Both watermark scalars therefore read "nothing happened" while the affected documents'
964
+ metadata is genuinely stale in the index.
965
+
966
+ The only signal that exists for that change is the event. So the server can optionally listen for
967
+ Paperless-ngx's own webhooks and re-index the documents a delivery names.
968
+
969
+ ### Turning it on
970
+
971
+ Set a port and a shared secret:
972
+
973
+ ```bash
974
+ PAPERLESS_MCP_WEBHOOK_PORT=8899
975
+ PAPERLESS_MCP_WEBHOOK_SECRET=<a-long-random-string>
976
+ ```
977
+
978
+ The receiver is **off by default** (`PAPERLESS_MCP_WEBHOOK_PORT=0`) and **refuses to start if the
979
+ secret is unset**, logging the reason instead. That refusal is deliberate: the endpoint causes
980
+ embedding work, and an unauthenticated trigger for it is a resource-abuse vector, so a warning line
981
+ nobody reads would not be a mitigation.
982
+
983
+ It is not a replacement for `sync_embeddings` or the background interval. A webhook only reports
984
+ events the process was alive to receive; a document added while the server was down produces no
985
+ delivery at all. The watermark still covers downtime, and the periodic metadata refresh still runs
986
+ as a backstop for a delivery that was attempted and never arrived (the sender retries a few times
987
+ and then gives up).
988
+
989
+ ### Creating the workflow in Paperless-ngx
990
+
991
+ Paperless-ngx fires webhooks through a *workflow*, which you create once. Nothing here creates it
992
+ for you — it is your instance to configure.
993
+
994
+ ```bash
995
+ curl -X POST "$PAPERLESS_URL/api/workflows/" \
996
+ -H "Authorization: Token $PAPERLESS_API_KEY" \
997
+ -H "Content-Type: application/json" \
998
+ -d '{
999
+ "name": "Keep the semantic index current",
1000
+ "enabled": true,
1001
+ "triggers": [
1002
+ { "type": 2, "matching_algorithm": 0 },
1003
+ { "type": 3, "matching_algorithm": 0 }
1004
+ ],
1005
+ "actions": [
1006
+ {
1007
+ "type": 4,
1008
+ "webhook": {
1009
+ "url": "http://172.17.0.1:8899/webhook/paperless",
1010
+ "use_params": true,
1011
+ "as_json": true,
1012
+ "params": { "id": "{{ doc_id }}" },
1013
+ "headers": { "X-Paperless-MCP-Secret": "<a-long-random-string>" }
1014
+ }
1015
+ }
1016
+ ]
1017
+ }'
1018
+ ```
1019
+
1020
+ Notes on that payload, each of which is a way to get it subtly wrong:
1021
+
1022
+ - **`type: 2` is "Document Added" and `type: 3` is "Document Updated"** on the trigger; **`type: 4`
1023
+ is "Webhook"** on the action. The trigger types are a separate numbering from the action types.
1024
+ - **`"matching_algorithm": 0`** means "no content filter", so the trigger matches every document. A
1025
+ non-zero value makes the trigger filter on document text, which is not what you want here.
1026
+ - The template uses **Jinja syntax** (`{{ doc_id }}`), not a different placeholder style, and is
1027
+ rendered before the request is sent. `use_params` with a `params` dictionary posts a form; add
1028
+ `"as_json": true` to post JSON instead. Both are parsed.
1029
+ - **The document id must be in the body.** A delivery that names no document still triggers a sync,
1030
+ but an untargeted one — correct, merely more expensive. The receiver cannot infer which document
1031
+ changed, because the payload carries no change description.
1032
+ - **The secret goes in `headers`.** The receiver also accepts it as a `Bearer` token if you would
1033
+ rather not add a custom header.
1034
+
1035
+ ### The container networking reality
1036
+
1037
+ **A Paperless-ngx running in a container cannot reach `127.0.0.1` on your host.** That address is
1038
+ the *container's* own loopback. If you leave `PAPERLESS_MCP_WEBHOOK_HOST` at its default and point
1039
+ the workflow at `127.0.0.1`, deliveries will fail with a connection error.
1040
+
1041
+ Two things have to be true instead:
1042
+
1043
+ 1. **The receiver binds an address the container can route to.** In a default Docker bridge setup
1044
+ the host is reachable from containers at the bridge gateway, commonly `172.17.0.1`. Set
1045
+ `PAPERLESS_MCP_WEBHOOK_HOST` to that (or `0.0.0.0` to listen on all interfaces) and use the same
1046
+ address in the workflow URL. If your containers run on a user-defined network, the gateway is a
1047
+ different address in that subnet; `docker network inspect <network>` shows it.
1048
+ 2. **Your host firewall permits the traffic.** Most default-DROP firewall configurations will
1049
+ silently swallow a connection from the bridge subnet, which looks exactly like the receiver not
1050
+ running. Permit the port from the bridge subnet specifically, not from everywhere.
1051
+
1052
+ Binding `0.0.0.0` widens the endpoint to every interface the host has. That is sometimes the only
1053
+ workable choice, and it is why the default is loopback: it should be a decision, not an inheritance.
1054
+
1055
+ ### How a delivery is handled
1056
+
1057
+ - The endpoint answers **`202 Accepted` before doing any work.** The sender times out after a few
1058
+ seconds and retries on failure, so acknowledging first is what turns those retries into harmless
1059
+ duplicates instead of repeated embedding runs.
1060
+ - Deliveries are **coalesced**: ids are collected and one sync runs after a quiet period. A bulk
1061
+ edit that produces hundreds of deliveries produces one run naming every affected document.
1062
+ - A delivery for a document that no longer exists is **counted as a failure and skipped**; the rest
1063
+ of the batch proceeds. A delivery can be in flight when its document is deleted, and that is not a
1064
+ reason to abandon the others.
1065
+ - **Deletions produce no delivery.** Paperless-ngx's trigger set has no delete type, so the
1066
+ document count in the watermark remains the only thing that notices one. This is stated plainly
1067
+ rather than worked around: there is no event to listen for.
1068
+
1069
+ ### Verifying it
1070
+
1071
+ Send one delivery yourself — the receiver accepts anything that presents the secret:
1072
+
1073
+ ```bash
1074
+ curl -i -X POST "http://127.0.0.1:8899/webhook/paperless" \
1075
+ -H "X-Paperless-MCP-Secret: <a-long-random-string>" \
1076
+ -H "Content-Type: application/json" \
1077
+ -d '{"id": 1}'
1078
+ ```
1079
+
1080
+ A `202` means it was accepted; the sync then runs after the debounce period, and its log line
1081
+ reports what it found. A `401` means the secret did not match, and a `413` means the body was over
1082
+ the 1 MB cap.
1083
+
787
1084
  ## Error Handling
788
1085
 
789
1086
  The server will show clear error messages if:
@@ -31,12 +31,7 @@ class PaperlessAPI {
31
31
  const isJson = !options.body || typeof options.body === "string";
32
32
  const mergedHeaders = Object.assign(Object.assign({ Authorization: `Token ${this.token}`, Accept: `application/json; version=${this.apiVersion}`, "Accept-Language": "en-US,en;q=0.9" }, (isJson ? { "Content-Type": "application/json" } : {})), (0, utils_1.headersToObject)(options.headers));
33
33
  try {
34
- const response = yield (0, axios_1.default)({
35
- url,
36
- method: options.method || "GET",
37
- headers: mergedHeaders,
38
- data: options.body,
39
- });
34
+ const response = yield (0, axios_1.default)(Object.assign({ url, method: options.method || "GET", headers: mergedHeaders, data: options.body }, (options.signal ? { signal: options.signal } : {})));
40
35
  const body = response.data;
41
36
  if (response.status < 200 || response.status >= 300) {
42
37
  console.error({
@@ -0,0 +1,28 @@
1
+ /**
2
+ * The exit code a finished build run must produce.
3
+ *
4
+ * ⚠️ EXTRACTED SO IT CAN BE FALSIFIED. `scripts/build-index.ts` runs `main()`
5
+ * on import, so nothing in the suite can execute it -- which meant the decision
6
+ * "does a partial build count as success?" had no test at all, and a deliberate
7
+ * mutation that silenced it stayed green. A one-line pure function is the
8
+ * smallest seam that makes the answer checkable.
9
+ *
10
+ * The distinction is not cosmetic. The run this replaces died at 1,727 of 2,466
11
+ * documents; the index was usable but missing 739, and the wrapper reported
12
+ * success. A caller that cannot tell "complete" from "partial" will keep
13
+ * serving an index that silently returns nothing for a third of the corpus.
14
+ *
15
+ * ⚠️ THIS FILE IS IN `src/` ON PURPOSE. It previously lived in
16
+ * `scripts/build-index.ts`, and the test importing it dragged one file from
17
+ * above `src/` into the compilation, which made TypeScript infer `rootDir` as
18
+ * the repository root and emit `build/src/...` instead of `build/...`. The
19
+ * package's `main` and `bin` both point at `build/index.js`, so the build
20
+ * silently produced a package whose entrypoint was the PREVIOUS build -- and
21
+ * `prepack` deletes `build` first, so a publish would have shipped nothing at
22
+ * all. 473 tests and a clean `tsc` exit both missed it, because the suite runs
23
+ * under ts-node where the output layout is irrelevant. Keeping this under
24
+ * `src/` is what holds the root still.
25
+ */
26
+ export declare function exitCodeForReport(report: {
27
+ failed: number;
28
+ }): number;
@@ -0,0 +1,31 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.exitCodeForReport = exitCodeForReport;
4
+ /**
5
+ * The exit code a finished build run must produce.
6
+ *
7
+ * ⚠️ EXTRACTED SO IT CAN BE FALSIFIED. `scripts/build-index.ts` runs `main()`
8
+ * on import, so nothing in the suite can execute it -- which meant the decision
9
+ * "does a partial build count as success?" had no test at all, and a deliberate
10
+ * mutation that silenced it stayed green. A one-line pure function is the
11
+ * smallest seam that makes the answer checkable.
12
+ *
13
+ * The distinction is not cosmetic. The run this replaces died at 1,727 of 2,466
14
+ * documents; the index was usable but missing 739, and the wrapper reported
15
+ * success. A caller that cannot tell "complete" from "partial" will keep
16
+ * serving an index that silently returns nothing for a third of the corpus.
17
+ *
18
+ * ⚠️ THIS FILE IS IN `src/` ON PURPOSE. It previously lived in
19
+ * `scripts/build-index.ts`, and the test importing it dragged one file from
20
+ * above `src/` into the compilation, which made TypeScript infer `rootDir` as
21
+ * the repository root and emit `build/src/...` instead of `build/...`. The
22
+ * package's `main` and `bin` both point at `build/index.js`, so the build
23
+ * silently produced a package whose entrypoint was the PREVIOUS build -- and
24
+ * `prepack` deletes `build` first, so a publish would have shipped nothing at
25
+ * all. 473 tests and a clean `tsc` exit both missed it, because the suite runs
26
+ * under ts-node where the output layout is irrelevant. Keeping this under
27
+ * `src/` is what holds the root still.
28
+ */
29
+ function exitCodeForReport(report) {
30
+ return report.failed > 0 ? 3 : 0;
31
+ }
@@ -0,0 +1,16 @@
1
+ export declare const CHUNK_SIZE = 2000;
2
+ export declare const CHUNK_OVERLAP = 200;
3
+ export declare const MAX_CHUNKS = 4000;
4
+ export interface ChunkResult {
5
+ chunks: string[];
6
+ truncated: boolean;
7
+ }
8
+ export declare function chunkText(text: string, maxChunks?: number): ChunkResult;
9
+ export declare function metadataText(doc: {
10
+ title: string;
11
+ created: string;
12
+ }, names: {
13
+ correspondent?: string;
14
+ documentType?: string;
15
+ tags: string[];
16
+ }): string;
@@ -0,0 +1,64 @@
1
+ "use strict";
2
+ // Text chunking for the paperless vector index.
3
+ //
4
+ // Algorithm ported from jaimemachado/paperless-mcp (src/search/chunk.ts at
5
+ // 50f7beec0d36804cab8fbcaa7b47f85e562e8061), kept near-verbatim. Two deliberate
6
+ // changes, both measured against our corpus:
7
+ //
8
+ // 1. metadataText() emitted its five scaffolding labels in SPANISH (title,
9
+ // type, sender, tags, date) at position 0 of every document, so those
10
+ // five tokens were embedded AND lexical-indexed across an English corpus.
11
+ // English now. Same structure, same ordering. No Spanish literal is kept
12
+ // in this file, not even in a comment, so a grep for the bug cannot hit
13
+ // this header and read like the bug survived.
14
+ // 2. MAX_CHUNKS 1000 -> 4000. 21 live documents exceed 2M chars (75,133,944
15
+ // chars total) and lost ~56% of their body at 1000 chunks; at 4000 no live
16
+ // document truncates, at a cost of ~20,700 extra chunks (~7% of the index).
17
+ //
18
+ // The stride is CHUNK_SIZE - CHUNK_OVERLAP = 1800, so consecutive chunks share
19
+ // exactly the trailing 200 characters of the previous one.
20
+ Object.defineProperty(exports, "__esModule", { value: true });
21
+ exports.MAX_CHUNKS = exports.CHUNK_OVERLAP = exports.CHUNK_SIZE = void 0;
22
+ exports.chunkText = chunkText;
23
+ exports.metadataText = metadataText;
24
+ exports.CHUNK_SIZE = 2000;
25
+ exports.CHUNK_OVERLAP = 200;
26
+ exports.MAX_CHUNKS = 4000;
27
+ const SKIPPED_TAG_PREFIXES = ["ai-", "via-"];
28
+ // A chunk has 2000 characters, about 512 tokens, and the overlap is 200
29
+ // characters. A document has at most maxChunks chunks (MAX_CHUNKS by default).
30
+ //
31
+ // truncated is true only when the cap was reached WITHOUT the text ending. A
32
+ // document that lands exactly on the cap is not truncated.
33
+ function chunkText(text, maxChunks = exports.MAX_CHUNKS) {
34
+ const normalized = text.replace(/\s+/g, " ").trim();
35
+ if (normalized === "") {
36
+ return { chunks: [], truncated: false };
37
+ }
38
+ const chunks = [];
39
+ let start = 0;
40
+ let reachedEnd = false;
41
+ while (start < normalized.length && chunks.length < maxChunks) {
42
+ chunks.push(normalized.slice(start, start + exports.CHUNK_SIZE));
43
+ if (start + exports.CHUNK_SIZE >= normalized.length) {
44
+ reachedEnd = true;
45
+ break;
46
+ }
47
+ start += exports.CHUNK_SIZE - exports.CHUNK_OVERLAP;
48
+ }
49
+ const truncated = !reachedEnd && chunks.length >= maxChunks;
50
+ return { chunks, truncated };
51
+ }
52
+ function metadataText(doc, names) {
53
+ const tags = names.tags.filter((tag) => !SKIPPED_TAG_PREFIXES.some((prefix) => tag.startsWith(prefix)));
54
+ const parts = [`Title: ${doc.title}.`];
55
+ if (names.documentType) {
56
+ parts.push(`Type: ${names.documentType}.`);
57
+ }
58
+ if (names.correspondent) {
59
+ parts.push(`Sender: ${names.correspondent}.`);
60
+ }
61
+ parts.push(`Tags: ${tags.join(", ")}.`);
62
+ parts.push(`Date: ${doc.created}.`);
63
+ return parts.join(" ");
64
+ }
@@ -0,0 +1,120 @@
1
+ /**
2
+ * Semantic-search configuration for the local SQLite index.
3
+ *
4
+ * Deliberate differences from the reference implementation this was ported
5
+ * from (`jaimemachado/paperless-mcp@50f7bee`):
6
+ *
7
+ * - A remote embedding provider IS reachable by configuration, because this
8
+ * package is published for general use and a third-party install must be
9
+ * able to point it at whichever provider it has. What protects a deployment
10
+ * that must not send text off the host is a flag it sets --
11
+ * `PAPERLESS_MCP_EMBEDDING_LOCAL_ONLY=true` -- not a restriction compiled
12
+ * into the library on its behalf.
13
+ * - There is NO `useHttp` gate. That gate makes semantic search dead on
14
+ * arrival for any deployment that launches this package as a stdio child
15
+ * with no CLI arguments, because `--http` is then never true. Nothing about
16
+ * a local sqlite index requires HTTP transport.
17
+ * - There is NO `DATABASE_URL`. The store is a file path.
18
+ * - Every value here has a working default, so "all four or none" would be
19
+ * meaningless. The rule instead is: any search variable present enables the
20
+ * feature; an invalid value refuses loudly rather than silently defaulting.
21
+ */
22
+ export declare const PREFIX_MODES: readonly ["nomic", "none", "e5"];
23
+ export type PrefixMode = (typeof PREFIX_MODES)[number];
24
+ export interface SearchConfig {
25
+ readonly embeddingUrl: string;
26
+ readonly model: string;
27
+ readonly prefixMode: PrefixMode;
28
+ readonly indexDir: string;
29
+ readonly maxChunks: number;
30
+ readonly syncBatchSize: number;
31
+ readonly syncIntervalMs: number;
32
+ /** Restrict the embedding endpoint to a literal loopback/RFC1918 address. */
33
+ readonly embeddingLocalOnly: boolean;
34
+ /** Bearer token for the embedding endpoint, when one is required. */
35
+ readonly embeddingApiKey: string | undefined;
36
+ /**
37
+ * Port the change-notification receiver listens on.
38
+ *
39
+ * 0 means OFF, and 0 is the default. The receiver is a listening socket in a
40
+ * process that was previously a pure stdio client; an install that did not
41
+ * ask for one must not acquire one by upgrading.
42
+ */
43
+ readonly webhookPort: number;
44
+ /** Shared secret the sender must present. Required whenever the port is on. */
45
+ readonly webhookSecret: string | undefined;
46
+ /** Address to bind. Loopback unless an operator widens it deliberately. */
47
+ readonly webhookHost: string;
48
+ /** Quiet period before a burst of events becomes one run. */
49
+ readonly webhookDebounceMs: number;
50
+ }
51
+ /**
52
+ * Rowid stride for the vector index: a chunk's rowid is
53
+ * `docId * CHUNK_ROWID_STRIDE + position`, so a document can hold at most
54
+ * `CHUNK_ROWID_STRIDE - 1` chunks before two documents collide.
55
+ *
56
+ * It lives in config.ts rather than store.ts so the CONFIG derives its own
57
+ * upper bound from the one number the store actually encodes with. A second
58
+ * literal would be a second source of truth, and the cost of the two
59
+ * disagreeing is paid mid-build: `PAPERLESS_MCP_MAX_CHUNKS=100000` was accepted
60
+ * by config and could only ever be rejected by `chunkRowid`, on document 2,000
61
+ * of a 58-minute run.
62
+ *
63
+ * A power of two, so a position decodes with a mask rather than a division that
64
+ * has to be exactly right.
65
+ */
66
+ export declare const CHUNK_ROWID_STRIDE = 8192;
67
+ /** The largest chunk count the stride can encode: positions 0 .. stride-1. */
68
+ export declare const MAX_CHUNKS_LIMIT: number;
69
+ export declare const DEFAULT_EMBEDDING_URL = "http://127.0.0.1:11434";
70
+ export declare const DEFAULT_EMBEDDING_MODEL = "nomic-embed-text";
71
+ export declare const DEFAULT_PREFIX_MODE: PrefixMode;
72
+ export declare const DEFAULT_MAX_CHUNKS = 4000;
73
+ export declare const DEFAULT_SYNC_BATCH_SIZE = 16;
74
+ /**
75
+ * Off by default. An install that must be able to prove indexed text does not
76
+ * leave the host turns it on; one that is pointing at a hosted provider must
77
+ * not have to fight a policy it never asked for.
78
+ */
79
+ export declare const DEFAULT_EMBEDDING_LOCAL_ONLY = false;
80
+ /** 0 means "no background sync". Opting in is explicit, never a default. */
81
+ export declare const DEFAULT_SYNC_INTERVAL_MS = 0;
82
+ /** 0 means "no change-notification receiver". Same reasoning as the interval. */
83
+ export declare const DEFAULT_WEBHOOK_PORT = 0;
84
+ /**
85
+ * Loopback, and it has to be said out loud rather than inherited from Node.
86
+ *
87
+ * A listener that defaults to every interface is reachable from anywhere the
88
+ * host is, which for an endpoint that triggers embedding work is a decision the
89
+ * operator should have made. A containerised sender cannot reach loopback and
90
+ * must therefore set this -- which is the intended friction.
91
+ */
92
+ export declare const DEFAULT_WEBHOOK_HOST = "127.0.0.1";
93
+ /** Long enough to absorb a bulk edit's burst, short enough to feel immediate. */
94
+ export declare const DEFAULT_WEBHOOK_DEBOUNCE_MS = 2000;
95
+ export declare function defaultIndexDir(): string;
96
+ /** Resolve a leading `~` without pulling in a shell. */
97
+ export declare function expandHome(input: string): string;
98
+ /**
99
+ * Returns the full configuration, or `undefined` when the feature is off.
100
+ *
101
+ * Off means: none of the search variables are set. Setting any one of them
102
+ * turns the feature on, and every set value is then validated strictly —
103
+ * a bad `EMBEDDING_PREFIX_MODE` throws rather than falling back to `nomic`,
104
+ * because a silent fallback here would silently change every stored vector.
105
+ */
106
+ export declare function searchConfig(env: Record<string, string | undefined>): SearchConfig | undefined;
107
+ /**
108
+ * Whether the receiver may start, and why not when it may not.
109
+ *
110
+ * ⚠️ A PORT WITHOUT A SECRET IS A CONFIGURATION ERROR, NOT A WARNING. The
111
+ * endpoint causes embedding work; an unauthenticated trigger for that is a
112
+ * resource-abuse vector, and a log line nobody reads is not a mitigation. So
113
+ * this returns a reason rather than a boolean -- the caller decides whether to
114
+ * log and continue (a background sync failing to start must not take down tool
115
+ * registration) or to fail, and either way the reason reaches the operator
116
+ * verbatim instead of being reconstructed.
117
+ */
118
+ export declare function webhookRefusal(config: SearchConfig): string | undefined;
119
+ /** The index file inside the configured directory. */
120
+ export declare function indexPath(config: SearchConfig): string;