@promptev/context-engine 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +202 -0
- package/NOTICE +17 -0
- package/README.md +112 -0
- package/dist/cli.js +11998 -0
- package/dist/cli.js.map +1 -0
- package/dist/config-Bl9U789m.d.cts +174 -0
- package/dist/config-Bt9bUQqU.d.ts +174 -0
- package/dist/embeddings-B-jZ42mk.d.cts +67 -0
- package/dist/embeddings-DaSdAZN3.d.ts +67 -0
- package/dist/express.cjs +3173 -0
- package/dist/express.cjs.map +1 -0
- package/dist/express.d.cts +24 -0
- package/dist/express.d.ts +24 -0
- package/dist/express.js +3170 -0
- package/dist/express.js.map +1 -0
- package/dist/fastify.cjs +3184 -0
- package/dist/fastify.cjs.map +1 -0
- package/dist/fastify.d.cts +16 -0
- package/dist/fastify.d.ts +16 -0
- package/dist/fastify.js +3181 -0
- package/dist/fastify.js.map +1 -0
- package/dist/governance-BDkcv4qZ.d.cts +79 -0
- package/dist/governance-XIScatRO.d.ts +79 -0
- package/dist/graph/index.cjs +1428 -0
- package/dist/graph/index.cjs.map +1 -0
- package/dist/graph/index.d.cts +104 -0
- package/dist/graph/index.d.ts +104 -0
- package/dist/graph/index.js +1413 -0
- package/dist/graph/index.js.map +1 -0
- package/dist/hono.cjs +3183 -0
- package/dist/hono.cjs.map +1 -0
- package/dist/hono.d.cts +39 -0
- package/dist/hono.d.ts +39 -0
- package/dist/hono.js +3179 -0
- package/dist/hono.js.map +1 -0
- package/dist/index.cjs +11731 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +851 -0
- package/dist/index.d.ts +851 -0
- package/dist/index.js +11676 -0
- package/dist/index.js.map +1 -0
- package/dist/mcp.cjs +181 -0
- package/dist/mcp.cjs.map +1 -0
- package/dist/mcp.d.cts +26 -0
- package/dist/mcp.d.ts +26 -0
- package/dist/mcp.js +179 -0
- package/dist/mcp.js.map +1 -0
- package/dist/migrations/sql/0001.sql +119 -0
- package/dist/migrations/sql/0002_graph.sql +48 -0
- package/dist/migrations/sql/0003_tools.sql +61 -0
- package/dist/migrations/sql/0004_acl_indexes.sql +4 -0
- package/dist/redaction-BmDSWJ7h.d.cts +98 -0
- package/dist/redaction-BmDSWJ7h.d.ts +98 -0
- package/dist/redaction-presidio.cjs +79 -0
- package/dist/redaction-presidio.cjs.map +1 -0
- package/dist/redaction-presidio.d.cts +22 -0
- package/dist/redaction-presidio.d.ts +22 -0
- package/dist/redaction-presidio.js +73 -0
- package/dist/redaction-presidio.js.map +1 -0
- package/dist/router-CrxZ2y_Z.d.ts +82 -0
- package/dist/router-OPgSoYAB.d.cts +82 -0
- package/dist/skills/context-engine/SKILL.md +160 -0
- package/package.json +184 -0
- package/src/migrations/sql/0001.sql +119 -0
- package/src/migrations/sql/0002_graph.sql +48 -0
- package/src/migrations/sql/0003_tools.sql +61 -0
- package/src/migrations/sql/0004_acl_indexes.sql +4 -0
- package/src/skills/context-engine/SKILL.md +160 -0
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: context-engine
|
|
3
|
+
description: Use when writing code against the @promptev/context-engine library — retrieval, ACLs/principals, ingestion, redaction, governed tools, or mounting its HTTP/MCP surfaces. Covers the access-control semantics that cause silent data leaks when guessed.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# context-engine (JavaScript / TypeScript)
|
|
7
|
+
|
|
8
|
+
Governed retrieval over your own Postgres. Access control is enforced inside
|
|
9
|
+
the SQL predicate of every retrieval leg, before ranking — not applied to
|
|
10
|
+
results afterwards.
|
|
11
|
+
|
|
12
|
+
Import `@promptev/context-engine`. Full guide: https://promptev.ai/documentation/context-engine/
|
|
13
|
+
|
|
14
|
+
## The one that leaks data
|
|
15
|
+
|
|
16
|
+
`principals` is a **trichotomy**, and the dangerous value is the one that
|
|
17
|
+
looks like an absence:
|
|
18
|
+
|
|
19
|
+
| value | meaning |
|
|
20
|
+
|---|---|
|
|
21
|
+
| `principals=TRUSTED` | Trusted caller — ACL filtering is **DISABLED**. Returns everything. Import it: `import { TRUSTED } from "@promptev/context-engine"`. |
|
|
22
|
+
| `principals=[]` | Anonymous — matches only documents with no ACL. |
|
|
23
|
+
| `principals=["group:hr", ...]` | Matches unrestricted documents plus any whose ACL overlaps the list. |
|
|
24
|
+
| `principals=null` | **DEPRECATED.** Same as `TRUSTED` today, warns, and raises in 1.0. |
|
|
25
|
+
|
|
26
|
+
**Never pass `null` because a request had no authenticated user.** That is the
|
|
27
|
+
opposite of what it means and it returns the entire corpus. Unauthenticated
|
|
28
|
+
means `[]`.
|
|
29
|
+
|
|
30
|
+
```ts
|
|
31
|
+
// WRONG — leaks every document to an anonymous caller
|
|
32
|
+
const principals = user ? user.groups : null;
|
|
33
|
+
|
|
34
|
+
// RIGHT
|
|
35
|
+
const principals = user ? user.groups : [];
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
The reason `null` is being retired: it is falsy, so it is what every accessor
|
|
39
|
+
degrades to — `user.groups || null`, a missing dict key, `user ? user.groups : null`.
|
|
40
|
+
The unauthenticated path silently became the maximum-privilege path. `TRUSTED`
|
|
41
|
+
cannot be produced by a null-coalesce; reaching it requires typing it.
|
|
42
|
+
|
|
43
|
+
Use `TRUSTED` for genuinely trusted in-process callers — a backfill script, an
|
|
44
|
+
admin job — never for anything derived from a request.
|
|
45
|
+
|
|
46
|
+
```ts
|
|
47
|
+
import { TRUSTED } from "@promptev/context-engine";
|
|
48
|
+
|
|
49
|
+
await engine.search(q, { principals: TRUSTED }); // explicit: skip ACL
|
|
50
|
+
await engine.search(q, { principals: [] }); // anonymous
|
|
51
|
+
await engine.search(q, { principals: user.groups }); // scoped
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Rules that prevent the other silent failures
|
|
55
|
+
|
|
56
|
+
- **Never read `principals` from a request body or query string.** On remote
|
|
57
|
+
surfaces they are injection-only: derive them server-side from the
|
|
58
|
+
authenticated session. A caller who can name their own principals has no
|
|
59
|
+
access control.
|
|
60
|
+
- **`auth` and `principals` are REQUIRED** on `createHonoRouter`,
|
|
61
|
+
`createExpressRouter`, `createFastifyPlugin` and `createMcpApp`. There is
|
|
62
|
+
no default. If you hit a `TypeError`, supply real ones — do **not** silence
|
|
63
|
+
it with `principals: () => null`, which disables access control on a public
|
|
64
|
+
port. Anonymous is `() => []`. Trusted is `() => TRUSTED`.
|
|
65
|
+
- **`acl=null` on `updateDocument` UNRESTRICTS the document.** "Leave it
|
|
66
|
+
alone" is expressed by omitting the argument (the `UNSET` sentinel), because
|
|
67
|
+
`null` already means unrestricted and cannot also mean absent. Same for
|
|
68
|
+
`metaData`, which merges rather than replaces.
|
|
69
|
+
- **Requires pgvector 0.8 or newer.** Below that, ACL-filtered vector search
|
|
70
|
+
silently loses recall — queries return fewer rows than match, or none, with
|
|
71
|
+
no error. The library cannot compensate.
|
|
72
|
+
- **`config.enableCodeExecution` is `false` on purpose.** `engine.compute()`
|
|
73
|
+
executes generated code. Turn it on only deliberately.
|
|
74
|
+
- Filing a document under an ACL the caller does not hold returns **403**, on
|
|
75
|
+
ingest and on PATCH alike.
|
|
76
|
+
|
|
77
|
+
## Search
|
|
78
|
+
|
|
79
|
+
```ts
|
|
80
|
+
const result = await engine.search("how much annual leave do I get", {
|
|
81
|
+
principals: ["group:hr"], // from the session, never the request body
|
|
82
|
+
sourceIds: ["hr-handbook"], // optional scope
|
|
83
|
+
topK: 8,
|
|
84
|
+
});
|
|
85
|
+
for (const hit of result.hits) {
|
|
86
|
+
console.log(hit.chunkText, hit.meta);
|
|
87
|
+
}
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Hybrid full-text + trigram + vector, fused with RRF. `mode: "graph"` switches
|
|
91
|
+
to the graph leg when the graph subsystem is configured.
|
|
92
|
+
|
|
93
|
+
## Ingest
|
|
94
|
+
|
|
95
|
+
```ts
|
|
96
|
+
const report = await engine.ingest({
|
|
97
|
+
content: fs.readFileSync("handbook.pdf"),
|
|
98
|
+
filename: "handbook.pdf",
|
|
99
|
+
sourceId: "hr-handbook",
|
|
100
|
+
acl: ["group:hr"], // omit for an unrestricted document
|
|
101
|
+
});
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Accepts `file`, `content` + `filename`, or `text`. Re-ingesting identical
|
|
105
|
+
content is deduplicated by hash; ACL changes on re-ingest are applied.
|
|
106
|
+
|
|
107
|
+
## Mounting the HTTP surface
|
|
108
|
+
|
|
109
|
+
Framework adapters live on **subpath exports**, not the main package entry
|
|
110
|
+
(so `import "@promptev/context-engine"` never pulls Hono/Express/Fastify):
|
|
111
|
+
|
|
112
|
+
```ts
|
|
113
|
+
import { createHonoRouter } from "@promptev/context-engine/hono";
|
|
114
|
+
import { TRUSTED } from "@promptev/context-engine";
|
|
115
|
+
|
|
116
|
+
app.route(
|
|
117
|
+
"/context",
|
|
118
|
+
createHonoRouter(engine, {
|
|
119
|
+
auth: async (c) => { /* raise / return 401 on anonymous */ },
|
|
120
|
+
principals: (c) => c.get("user")?.groups ?? [],
|
|
121
|
+
}),
|
|
122
|
+
);
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
`createExpressRouter` (`@promptev/context-engine/express`),
|
|
126
|
+
`createFastifyPlugin` (`@promptev/context-engine/fastify`) and
|
|
127
|
+
`createMcpApp` (`@promptev/context-engine/mcp`) take the same two
|
|
128
|
+
arguments. MCP has no request object: `principals` is a zero-argument
|
|
129
|
+
callable resolved fresh on every tool call. The CLI `context-engine mcp`
|
|
130
|
+
binds `127.0.0.1` and answers as anonymous (`principals=[]`).
|
|
131
|
+
|
|
132
|
+
Not on a supported framework? Call `engine.search()` / `engine.ingest()`
|
|
133
|
+
from your own handler — the routers are thin wrappers over the same core.
|
|
134
|
+
|
|
135
|
+
## Lifecycle
|
|
136
|
+
|
|
137
|
+
Construction is lazy; close the engine to tear down only what was built. A
|
|
138
|
+
process-lifetime singleton can skip it, but anything that builds engines
|
|
139
|
+
repeatedly (worker per job, test per case) must close them or it accumulates
|
|
140
|
+
connection pools.
|
|
141
|
+
|
|
142
|
+
## Redaction
|
|
143
|
+
|
|
144
|
+
Rules with `applyAt: "ingest"` run **before chunks are embedded**, so masked
|
|
145
|
+
text is what reaches the embedding provider. `applyAt: "output"` masks on the
|
|
146
|
+
way out and leaves stored text intact. Use `unless: [...]` to let named
|
|
147
|
+
principals see through a rule.
|
|
148
|
+
|
|
149
|
+
Presidio NER is an HTTP adapter (`@promptev/context-engine/presidio`): set
|
|
150
|
+
`CE_PRESIDIO_URL` or `PRESIDIO_URL` to a Presidio Analyzer endpoint.
|
|
151
|
+
|
|
152
|
+
## Don't
|
|
153
|
+
|
|
154
|
+
- Don't add a `WHERE` clause of your own on top of a retrieval call to filter
|
|
155
|
+
by tenant — pass `principals`/`sourceIds` and let it go into the indexed
|
|
156
|
+
predicate. Filtering after the fact is what loses recall.
|
|
157
|
+
- Don't cache a `SearchResult` across users. Results are ACL-scoped to the
|
|
158
|
+
caller who requested them.
|
|
159
|
+
- Don't expect `engine.search()` to enforce anything about *writes*; ingest
|
|
160
|
+
and PATCH authorisation is separate and already handled.
|
package/package.json
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@promptev/context-engine",
|
|
3
|
+
"version": "0.0.0",
|
|
4
|
+
"description": "Promptev Context Engine — language-agnostic ingestion + hybrid retrieval (TypeScript)",
|
|
5
|
+
"license": "Apache-2.0",
|
|
6
|
+
"author": "Promptev Inc",
|
|
7
|
+
"homepage": "https://promptev.ai/documentation/context-engine/",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/promptev/context-engine.git",
|
|
11
|
+
"directory": "js"
|
|
12
|
+
},
|
|
13
|
+
"bugs": {
|
|
14
|
+
"url": "https://github.com/promptev/context-engine/issues"
|
|
15
|
+
},
|
|
16
|
+
"keywords": [
|
|
17
|
+
"rag",
|
|
18
|
+
"retrieval",
|
|
19
|
+
"hybrid-search",
|
|
20
|
+
"graph-rag",
|
|
21
|
+
"vector-search",
|
|
22
|
+
"embeddings",
|
|
23
|
+
"llm",
|
|
24
|
+
"pii-redaction",
|
|
25
|
+
"mcp"
|
|
26
|
+
],
|
|
27
|
+
"type": "module",
|
|
28
|
+
"main": "./dist/index.cjs",
|
|
29
|
+
"module": "./dist/index.js",
|
|
30
|
+
"types": "./dist/index.d.ts",
|
|
31
|
+
"bin": {
|
|
32
|
+
"context-engine": "dist/cli.js"
|
|
33
|
+
},
|
|
34
|
+
"exports": {
|
|
35
|
+
".": {
|
|
36
|
+
"types": "./dist/index.d.ts",
|
|
37
|
+
"import": "./dist/index.js",
|
|
38
|
+
"require": "./dist/index.cjs"
|
|
39
|
+
},
|
|
40
|
+
"./hono": {
|
|
41
|
+
"types": "./dist/hono.d.ts",
|
|
42
|
+
"import": "./dist/hono.js",
|
|
43
|
+
"require": "./dist/hono.cjs"
|
|
44
|
+
},
|
|
45
|
+
"./express": {
|
|
46
|
+
"types": "./dist/express.d.ts",
|
|
47
|
+
"import": "./dist/express.js",
|
|
48
|
+
"require": "./dist/express.cjs"
|
|
49
|
+
},
|
|
50
|
+
"./fastify": {
|
|
51
|
+
"types": "./dist/fastify.d.ts",
|
|
52
|
+
"import": "./dist/fastify.js",
|
|
53
|
+
"require": "./dist/fastify.cjs"
|
|
54
|
+
},
|
|
55
|
+
"./mcp": {
|
|
56
|
+
"types": "./dist/mcp.d.ts",
|
|
57
|
+
"import": "./dist/mcp.js",
|
|
58
|
+
"require": "./dist/mcp.cjs"
|
|
59
|
+
},
|
|
60
|
+
"./graph": {
|
|
61
|
+
"types": "./dist/graph/index.d.ts",
|
|
62
|
+
"import": "./dist/graph/index.js",
|
|
63
|
+
"require": "./dist/graph/index.cjs"
|
|
64
|
+
},
|
|
65
|
+
"./presidio": {
|
|
66
|
+
"types": "./dist/redaction-presidio.d.ts",
|
|
67
|
+
"import": "./dist/redaction-presidio.js",
|
|
68
|
+
"require": "./dist/redaction-presidio.cjs"
|
|
69
|
+
}
|
|
70
|
+
},
|
|
71
|
+
"files": [
|
|
72
|
+
"dist",
|
|
73
|
+
"src/migrations/sql",
|
|
74
|
+
"src/skills",
|
|
75
|
+
"LICENSE.md",
|
|
76
|
+
"NOTICE"
|
|
77
|
+
],
|
|
78
|
+
"engines": {
|
|
79
|
+
"node": ">=22"
|
|
80
|
+
},
|
|
81
|
+
"scripts": {
|
|
82
|
+
"build": "tsup",
|
|
83
|
+
"dev": "tsup --watch",
|
|
84
|
+
"test": "vitest run",
|
|
85
|
+
"test:watch": "vitest",
|
|
86
|
+
"typecheck": "tsc --noEmit",
|
|
87
|
+
"lint": "biome check src tests",
|
|
88
|
+
"lint:fix": "biome check --write src tests",
|
|
89
|
+
"context-engine": "node dist/cli.js"
|
|
90
|
+
},
|
|
91
|
+
"dependencies": {
|
|
92
|
+
"cheerio": "^1.1.0",
|
|
93
|
+
"commander": "^14.0.0",
|
|
94
|
+
"exceljs": "^4.4.0",
|
|
95
|
+
"franc": "^6.2.0",
|
|
96
|
+
"js-tiktoken": "^1.0.21",
|
|
97
|
+
"jsonrepair": "^3.13.0",
|
|
98
|
+
"jszip": "^3.10.1",
|
|
99
|
+
"mammoth": "^1.9.1",
|
|
100
|
+
"openai": "^5.12.0",
|
|
101
|
+
"postal-mime": "^2.4.4",
|
|
102
|
+
"unpdf": "^1.1.0",
|
|
103
|
+
"zod": "^3.25.0"
|
|
104
|
+
},
|
|
105
|
+
"peerDependencies": {
|
|
106
|
+
"@aws-sdk/client-bedrock-runtime": "^3.800.0",
|
|
107
|
+
"@fastify/multipart": "^9.0.0",
|
|
108
|
+
"@google/genai": "^1.24.0",
|
|
109
|
+
"@modelcontextprotocol/sdk": "^1.12.0",
|
|
110
|
+
"danfojs-node": "^1.1.2",
|
|
111
|
+
"express": "^5.1.0",
|
|
112
|
+
"fastify": "^5.3.0",
|
|
113
|
+
"graphology": "^0.25.4",
|
|
114
|
+
"graphology-communities-louvain": "^2.0.1",
|
|
115
|
+
"hono": "^4.7.0",
|
|
116
|
+
"isolated-vm": "^5.0.0",
|
|
117
|
+
"neo4j-driver": "^5.28.0",
|
|
118
|
+
"pdfjs-dist": "^5.3.0",
|
|
119
|
+
"pg": "^8.16.0",
|
|
120
|
+
"sharp": "^0.34.0",
|
|
121
|
+
"tesseract.js": "^6.0.0"
|
|
122
|
+
},
|
|
123
|
+
"peerDependenciesMeta": {
|
|
124
|
+
"@aws-sdk/client-bedrock-runtime": {
|
|
125
|
+
"optional": true
|
|
126
|
+
},
|
|
127
|
+
"@fastify/multipart": {
|
|
128
|
+
"optional": true
|
|
129
|
+
},
|
|
130
|
+
"@google/genai": {
|
|
131
|
+
"optional": true
|
|
132
|
+
},
|
|
133
|
+
"@modelcontextprotocol/sdk": {
|
|
134
|
+
"optional": true
|
|
135
|
+
},
|
|
136
|
+
"danfojs-node": {
|
|
137
|
+
"optional": true
|
|
138
|
+
},
|
|
139
|
+
"express": {
|
|
140
|
+
"optional": true
|
|
141
|
+
},
|
|
142
|
+
"fastify": {
|
|
143
|
+
"optional": true
|
|
144
|
+
},
|
|
145
|
+
"graphology": {
|
|
146
|
+
"optional": true
|
|
147
|
+
},
|
|
148
|
+
"graphology-communities-louvain": {
|
|
149
|
+
"optional": true
|
|
150
|
+
},
|
|
151
|
+
"hono": {
|
|
152
|
+
"optional": true
|
|
153
|
+
},
|
|
154
|
+
"isolated-vm": {
|
|
155
|
+
"optional": true
|
|
156
|
+
},
|
|
157
|
+
"neo4j-driver": {
|
|
158
|
+
"optional": true
|
|
159
|
+
},
|
|
160
|
+
"pdfjs-dist": {
|
|
161
|
+
"optional": true
|
|
162
|
+
},
|
|
163
|
+
"pg": {
|
|
164
|
+
"optional": true
|
|
165
|
+
},
|
|
166
|
+
"sharp": {
|
|
167
|
+
"optional": true
|
|
168
|
+
},
|
|
169
|
+
"tesseract.js": {
|
|
170
|
+
"optional": true
|
|
171
|
+
}
|
|
172
|
+
},
|
|
173
|
+
"devDependencies": {
|
|
174
|
+
"@biomejs/biome": "^2.1.0",
|
|
175
|
+
"@testcontainers/postgresql": "^11.0.0",
|
|
176
|
+
"@types/express": "^5.0.0",
|
|
177
|
+
"@types/node": "^22.16.0",
|
|
178
|
+
"@types/pg": "^8.15.0",
|
|
179
|
+
"pg": "^8.23.0",
|
|
180
|
+
"tsup": "^8.5.0",
|
|
181
|
+
"typescript": "^5.8.0",
|
|
182
|
+
"vitest": "^3.2.0"
|
|
183
|
+
}
|
|
184
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
-- 0001_initial — documents, chunks, meta, structured_keys
|
|
2
|
+
-- dim is substituted by the migrator (default 1536)
|
|
3
|
+
|
|
4
|
+
CREATE EXTENSION IF NOT EXISTS vector;
|
|
5
|
+
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
|
6
|
+
CREATE EXTENSION IF NOT EXISTS unaccent;
|
|
7
|
+
|
|
8
|
+
CREATE OR REPLACE FUNCTION immutable_unaccent(text)
|
|
9
|
+
RETURNS text
|
|
10
|
+
LANGUAGE sql
|
|
11
|
+
IMMUTABLE
|
|
12
|
+
STRICT
|
|
13
|
+
PARALLEL SAFE
|
|
14
|
+
AS $fn$
|
|
15
|
+
SELECT unaccent($1)
|
|
16
|
+
$fn$;
|
|
17
|
+
|
|
18
|
+
CREATE TABLE context_engine_documents (
|
|
19
|
+
id UUID PRIMARY KEY,
|
|
20
|
+
source_id TEXT,
|
|
21
|
+
external_id TEXT,
|
|
22
|
+
name TEXT,
|
|
23
|
+
description TEXT,
|
|
24
|
+
meta_data JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
25
|
+
acl TEXT[],
|
|
26
|
+
mode TEXT NOT NULL DEFAULT 'hybrid',
|
|
27
|
+
mime_type TEXT,
|
|
28
|
+
lang TEXT,
|
|
29
|
+
text TEXT,
|
|
30
|
+
text_search TSVECTOR GENERATED ALWAYS AS (
|
|
31
|
+
to_tsvector('simple'::regconfig,
|
|
32
|
+
left(coalesce(name,'') || ' ' || coalesce(description,'') || ' ' || coalesce(text,''), 500000))
|
|
33
|
+
) STORED,
|
|
34
|
+
modified_time TIMESTAMPTZ,
|
|
35
|
+
revision TEXT,
|
|
36
|
+
ingest_hash BYTEA,
|
|
37
|
+
status TEXT NOT NULL DEFAULT 'queued',
|
|
38
|
+
error TEXT,
|
|
39
|
+
started_at TIMESTAMPTZ,
|
|
40
|
+
completed_at TIMESTAMPTZ,
|
|
41
|
+
document_type TEXT,
|
|
42
|
+
structured_data JSONB,
|
|
43
|
+
structured_keys TEXT[],
|
|
44
|
+
structured_quality JSONB,
|
|
45
|
+
extraction_version TEXT,
|
|
46
|
+
media_bytes BYTEA,
|
|
47
|
+
media_size BIGINT,
|
|
48
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
49
|
+
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
CREATE INDEX ix_context_engine_documents_source_id ON context_engine_documents (source_id);
|
|
53
|
+
CREATE INDEX ix_context_engine_documents_status ON context_engine_documents (status);
|
|
54
|
+
CREATE INDEX ix_context_engine_documents_meta_data_gin ON context_engine_documents USING gin (meta_data);
|
|
55
|
+
CREATE INDEX ix_context_engine_documents_text_search ON context_engine_documents USING gin (text_search);
|
|
56
|
+
|
|
57
|
+
CREATE UNIQUE INDEX ix_context_engine_documents_uniq_source_external
|
|
58
|
+
ON context_engine_documents (COALESCE(source_id, ''), external_id)
|
|
59
|
+
WHERE external_id IS NOT NULL;
|
|
60
|
+
|
|
61
|
+
CREATE UNIQUE INDEX ix_context_engine_documents_uniq_source_name
|
|
62
|
+
ON context_engine_documents (COALESCE(source_id, ''), name)
|
|
63
|
+
WHERE external_id IS NULL;
|
|
64
|
+
|
|
65
|
+
CREATE TABLE context_engine_chunks (
|
|
66
|
+
id UUID PRIMARY KEY,
|
|
67
|
+
document_id UUID NOT NULL REFERENCES context_engine_documents(id) ON DELETE CASCADE,
|
|
68
|
+
source_id TEXT,
|
|
69
|
+
acl TEXT[],
|
|
70
|
+
idx INTEGER,
|
|
71
|
+
text TEXT,
|
|
72
|
+
meta_data JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
73
|
+
text_trgm_norm TEXT GENERATED ALWAYS AS (coalesce(text, '')) STORED,
|
|
74
|
+
text_search TSVECTOR GENERATED ALWAYS AS (
|
|
75
|
+
to_tsvector('simple'::regconfig, left(coalesce(text,''), 500000))
|
|
76
|
+
) STORED,
|
|
77
|
+
lang TEXT,
|
|
78
|
+
embedding vector(__DIM__),
|
|
79
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
80
|
+
);
|
|
81
|
+
|
|
82
|
+
CREATE INDEX ix_context_engine_chunks_document_id ON context_engine_chunks (document_id);
|
|
83
|
+
CREATE INDEX ix_context_engine_chunks_source_id ON context_engine_chunks (source_id);
|
|
84
|
+
CREATE INDEX ix_context_engine_chunks_text_search ON context_engine_chunks USING gin (text_search) WHERE text IS NOT NULL;
|
|
85
|
+
CREATE INDEX ix_context_engine_chunks_text_trgm_norm ON context_engine_chunks USING gin (text_trgm_norm gin_trgm_ops) WHERE text IS NOT NULL;
|
|
86
|
+
CREATE INDEX ix_context_engine_chunks_embedding_hnsw ON context_engine_chunks USING hnsw (embedding vector_cosine_ops) WHERE embedding IS NOT NULL;
|
|
87
|
+
|
|
88
|
+
CREATE TABLE context_engine_meta (
|
|
89
|
+
id INTEGER PRIMARY KEY,
|
|
90
|
+
embedding_provider TEXT,
|
|
91
|
+
embedding_model TEXT,
|
|
92
|
+
embedding_dim INTEGER,
|
|
93
|
+
schema_version INTEGER NOT NULL DEFAULT 1,
|
|
94
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
95
|
+
CONSTRAINT chk_context_engine_meta_singleton CHECK (id = 1)
|
|
96
|
+
);
|
|
97
|
+
|
|
98
|
+
CREATE TABLE context_engine_structured_keys (
|
|
99
|
+
id UUID PRIMARY KEY,
|
|
100
|
+
source_id TEXT,
|
|
101
|
+
doc_type TEXT NOT NULL,
|
|
102
|
+
key_norm TEXT NOT NULL,
|
|
103
|
+
synonyms TEXT[] NOT NULL DEFAULT '{}',
|
|
104
|
+
data_type TEXT NOT NULL,
|
|
105
|
+
example_values TEXT[],
|
|
106
|
+
frequency INTEGER NOT NULL DEFAULT 1,
|
|
107
|
+
embedding vector(__DIM__),
|
|
108
|
+
co_occurrence_fields JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
109
|
+
value_stats JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
110
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
111
|
+
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
112
|
+
CONSTRAINT uq_context_engine_structured_keys_norm UNIQUE (source_id, doc_type, key_norm)
|
|
113
|
+
);
|
|
114
|
+
|
|
115
|
+
CREATE INDEX ix_context_engine_structured_keys_source_doctype ON context_engine_structured_keys (source_id, doc_type);
|
|
116
|
+
CREATE INDEX ix_context_engine_structured_keys_synonyms_gin ON context_engine_structured_keys USING gin (synonyms);
|
|
117
|
+
CREATE INDEX ix_context_engine_structured_keys_co_occurrence_gin ON context_engine_structured_keys USING gin (co_occurrence_fields);
|
|
118
|
+
CREATE INDEX ix_context_engine_structured_keys_value_stats_gin ON context_engine_structured_keys USING gin (value_stats);
|
|
119
|
+
CREATE INDEX ix_context_engine_structured_keys_embedding_hnsw ON context_engine_structured_keys USING hnsw (embedding vector_cosine_ops) WHERE embedding IS NOT NULL;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
-- 0002_graph — entities, relationships, chunk_entities, communities
|
|
2
|
+
|
|
3
|
+
CREATE TABLE context_engine_entities (
|
|
4
|
+
id UUID PRIMARY KEY,
|
|
5
|
+
name TEXT NOT NULL,
|
|
6
|
+
normalized_name TEXT NOT NULL,
|
|
7
|
+
type TEXT NOT NULL,
|
|
8
|
+
frequency INTEGER NOT NULL DEFAULT 1,
|
|
9
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
10
|
+
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
11
|
+
CONSTRAINT uq_context_engine_entities_normalized_name UNIQUE (normalized_name)
|
|
12
|
+
);
|
|
13
|
+
CREATE INDEX ix_context_engine_entities_type ON context_engine_entities (type);
|
|
14
|
+
|
|
15
|
+
CREATE TABLE context_engine_entity_relationships (
|
|
16
|
+
id UUID PRIMARY KEY,
|
|
17
|
+
source_entity_id UUID NOT NULL REFERENCES context_engine_entities(id) ON DELETE CASCADE,
|
|
18
|
+
target_entity_id UUID NOT NULL REFERENCES context_engine_entities(id) ON DELETE CASCADE,
|
|
19
|
+
category TEXT NOT NULL,
|
|
20
|
+
label TEXT NOT NULL,
|
|
21
|
+
evidence TEXT,
|
|
22
|
+
chunk_id UUID REFERENCES context_engine_chunks(id) ON DELETE CASCADE,
|
|
23
|
+
confidence DOUBLE PRECISION NOT NULL DEFAULT 1.0,
|
|
24
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
25
|
+
);
|
|
26
|
+
CREATE INDEX ix_context_engine_entity_rel_source ON context_engine_entity_relationships (source_entity_id);
|
|
27
|
+
CREATE INDEX ix_context_engine_entity_rel_target ON context_engine_entity_relationships (target_entity_id);
|
|
28
|
+
CREATE INDEX ix_context_engine_entity_rel_category ON context_engine_entity_relationships (category);
|
|
29
|
+
CREATE INDEX ix_context_engine_entity_rel_chunk ON context_engine_entity_relationships (chunk_id);
|
|
30
|
+
|
|
31
|
+
CREATE TABLE context_engine_chunk_entities (
|
|
32
|
+
chunk_id UUID NOT NULL REFERENCES context_engine_chunks(id) ON DELETE CASCADE,
|
|
33
|
+
entity_id UUID NOT NULL REFERENCES context_engine_entities(id) ON DELETE CASCADE,
|
|
34
|
+
PRIMARY KEY (chunk_id, entity_id)
|
|
35
|
+
);
|
|
36
|
+
CREATE INDEX ix_context_engine_chunk_entities_entity ON context_engine_chunk_entities (entity_id);
|
|
37
|
+
|
|
38
|
+
CREATE TABLE context_engine_communities (
|
|
39
|
+
id UUID PRIMARY KEY,
|
|
40
|
+
level INTEGER NOT NULL DEFAULT 0,
|
|
41
|
+
entity_ids JSONB NOT NULL DEFAULT '[]'::jsonb,
|
|
42
|
+
entity_count INTEGER NOT NULL DEFAULT 0,
|
|
43
|
+
relationship_count INTEGER NOT NULL DEFAULT 0,
|
|
44
|
+
summary TEXT,
|
|
45
|
+
embedding vector(__DIM__),
|
|
46
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
47
|
+
);
|
|
48
|
+
CREATE INDEX ix_context_engine_communities_level ON context_engine_communities (level);
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
-- 0003_tools — tools, approvals, calls, oauth pending
|
|
2
|
+
|
|
3
|
+
CREATE TABLE context_engine_tools (
|
|
4
|
+
id UUID PRIMARY KEY,
|
|
5
|
+
name TEXT NOT NULL,
|
|
6
|
+
kind TEXT NOT NULL,
|
|
7
|
+
description TEXT NOT NULL DEFAULT '',
|
|
8
|
+
source_id TEXT,
|
|
9
|
+
acl TEXT[],
|
|
10
|
+
config_encrypted TEXT,
|
|
11
|
+
requires_approval BOOLEAN NOT NULL DEFAULT false,
|
|
12
|
+
approval_policy JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
13
|
+
enabled BOOLEAN NOT NULL DEFAULT true,
|
|
14
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
15
|
+
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
16
|
+
);
|
|
17
|
+
CREATE INDEX ix_context_engine_tools_source_id ON context_engine_tools (source_id);
|
|
18
|
+
|
|
19
|
+
CREATE TABLE context_engine_tool_approvals (
|
|
20
|
+
id UUID PRIMARY KEY,
|
|
21
|
+
tool_name TEXT NOT NULL,
|
|
22
|
+
tool_args_frozen JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
23
|
+
source_id TEXT,
|
|
24
|
+
principals JSONB,
|
|
25
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
26
|
+
approver TEXT,
|
|
27
|
+
approver_meta JSONB,
|
|
28
|
+
expires_at TIMESTAMPTZ,
|
|
29
|
+
resolved_at TIMESTAMPTZ,
|
|
30
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
31
|
+
);
|
|
32
|
+
CREATE INDEX ix_context_engine_tool_approvals_status ON context_engine_tool_approvals (status);
|
|
33
|
+
|
|
34
|
+
CREATE TABLE context_engine_tool_calls (
|
|
35
|
+
id UUID PRIMARY KEY,
|
|
36
|
+
tool_id UUID REFERENCES context_engine_tools(id) ON DELETE SET NULL,
|
|
37
|
+
tool_name TEXT NOT NULL,
|
|
38
|
+
kind TEXT NOT NULL,
|
|
39
|
+
actor_type TEXT,
|
|
40
|
+
actor_id TEXT,
|
|
41
|
+
source TEXT,
|
|
42
|
+
input_args JSONB NOT NULL DEFAULT '{}'::jsonb,
|
|
43
|
+
output_result JSONB,
|
|
44
|
+
output_truncated BOOLEAN NOT NULL DEFAULT false,
|
|
45
|
+
success BOOLEAN,
|
|
46
|
+
error_message TEXT,
|
|
47
|
+
duration_ms INTEGER,
|
|
48
|
+
units INTEGER,
|
|
49
|
+
approval_id UUID REFERENCES context_engine_tool_approvals(id) ON DELETE SET NULL,
|
|
50
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
51
|
+
);
|
|
52
|
+
CREATE INDEX ix_context_engine_tool_calls_tool_id ON context_engine_tool_calls (tool_id);
|
|
53
|
+
CREATE INDEX ix_context_engine_tool_calls_created_at ON context_engine_tool_calls (created_at);
|
|
54
|
+
|
|
55
|
+
CREATE TABLE context_engine_oauth_pending (
|
|
56
|
+
state TEXT PRIMARY KEY,
|
|
57
|
+
data JSONB NOT NULL,
|
|
58
|
+
expires_at TIMESTAMPTZ NOT NULL,
|
|
59
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
60
|
+
);
|
|
61
|
+
CREATE INDEX ix_context_engine_oauth_pending_expires_at ON context_engine_oauth_pending (expires_at);
|