@anmol-srv/sigil 0.18.2 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -25
- package/dist/cli.js +1191 -790
- package/dist/daemon.js +664 -302
- package/dist/hooks/post-tool-use.js +1 -69
- package/dist/hooks/session-end.js +13 -236
- package/dist/hooks/stop.js +36 -246
- package/dist/hooks/user-prompt-submit.js +25 -288
- package/dist/mcp/worker-server.js +6 -0
- package/dist/npm-entry.js +10 -0
- package/dist/server.js +64 -56
- package/install.sh +207 -0
- package/integrations/hermes/README.md +30 -24
- package/integrations/hermes/plugin/README.md +1 -1
- package/integrations/hermes/plugin/__init__.py +68 -24
- package/knexfile.js +19 -22
- package/package.json +9 -28
- package/prompts/audm-decision.md +26 -1
- package/prompts/entity-extraction.md +32 -30
- package/prompts/graph-extraction.md +20 -2
- package/src/db/drivers/index.js +38 -15
- package/src/db/migrations/20260629120000_add-trace-event-attribution.cjs +29 -0
- package/src/db/migrations/20260629120001_add-llm-log-session-correlation.cjs +29 -0
- package/src/db/migrations/20260806120000_add-document-content.cjs +27 -0
- package/src/db/migrations/20260806120001_llm-log-model-nullable.cjs +36 -0
- package/src/db/migrations/20260811120000_add-relation-provenance.cjs +42 -0
- package/src/db/migrations/20260811130000_add-fact-visibility.cjs +67 -0
- package/src/db/migrations/20260813090000_rescore-defaulted-vital.cjs +57 -0
- package/src/db/migrations/20260814120000_create-ingestion-job.cjs +52 -0
- package/src/db/migrations/20260814120100_add-llm-log-queued-ms.cjs +8 -0
- package/src/db/migrations/20260814120200_add-active-fact-content-hash-index.cjs +16 -0
- package/src/db/migrations/20260814120300_create-relation-candidate-judgment.cjs +25 -0
- package/src/gui/web/app.css +851 -84
- package/src/gui/web/app.js +796 -383
- package/src/gui/web/cmdk.js +238 -0
- package/src/gui/web/components.js +64 -0
- package/src/gui/web/design/colors_and_type.css +6 -1
- package/src/gui/web/graph/GraphIsland.jsx +450 -0
- package/src/gui/web/graph/cluster.js +305 -0
- package/src/gui/web/graph/cluster.test.js +219 -0
- package/src/gui/web/health.js +117 -0
- package/src/gui/web/health.test.js +159 -0
- package/src/gui/web/icons.js +65 -0
- package/src/gui/web/index.html +310 -115
- package/src/gui/web/settings.js +386 -0
- package/src/gui/web/setup.js +2 -2
- package/src/gui/web/toast.js +6 -3
- package/src/gui/web/vendor/graph-island.js +52 -0
|
@@ -1,37 +1,39 @@
|
|
|
1
|
-
You are extracting
|
|
2
|
-
A "topic" is a distinct concept, technology, system, process, or subject referenced in the facts.
|
|
1
|
+
You are extracting the entities from a set of facts in a personal knowledge base: the distinct topics, concepts, technologies, systems, products, people, or named things the facts are about.
|
|
3
2
|
|
|
4
|
-
|
|
3
|
+
Extract entities ONLY. Do not infer relationships between them — that is derived separately, from which entities actually co-occur across the whole store.
|
|
5
4
|
|
|
6
|
-
|
|
7
|
-
- Use canonical names: "normalization" not "database normalization concepts", "React hooks" not "React.js hooks pattern".
|
|
8
|
-
- If two facts mention the same topic with different wording, extract it once with the canonical name.
|
|
9
|
-
- Include a brief description (1 sentence) for context.
|
|
10
|
-
- Do NOT extract generic terms like "programming", "coding", "software". Be specific.
|
|
11
|
-
- Do NOT extract people names or document titles — those are handled separately.
|
|
12
|
-
- Topics should be reusable across documents — "database indexing" not "the indexing discussion in doc 12".
|
|
5
|
+
## Entities
|
|
13
6
|
|
|
14
|
-
|
|
7
|
+
- Extract 3-8 meaningful, distinct entities. Use canonical names, lowercase: "react hooks" not "the React.js hooks pattern".
|
|
8
|
+
- Include a one-sentence description giving context.
|
|
9
|
+
- Do NOT extract generic terms ("programming", "software", "data").
|
|
10
|
+
- If two facts mention the same thing with different wording, list it once.
|
|
15
11
|
|
|
16
|
-
|
|
12
|
+
### The knowledge-base owner — the one exception to "no generic terms"
|
|
13
|
+
|
|
14
|
+
Many facts describe the person who owns this knowledge base, and they almost never name them. They are written in the third person ("User prefers concise responses", "User's name is Anmol") or the first person ("I use Postgres", "my editor is Neovim").
|
|
15
|
+
|
|
16
|
+
That person is a real entity — usually the most connected one in the graph — so:
|
|
17
|
+
|
|
18
|
+
- Refer to them with the exact name `user`, always lowercase, and list them once with a description like "the owner of this knowledge base".
|
|
19
|
+
- Do this even though the fact never spells out their name.
|
|
20
|
+
|
|
21
|
+
`user` is the ONLY generic term you may extract.
|
|
22
|
+
|
|
23
|
+
### Rename contexts — ALWAYS extract BOTH names
|
|
24
|
+
|
|
25
|
+
When the source mentions a rename ("X is now named Y", "X was renamed to Y", "X used to be called Y") extract **both** the old and new names as separate entities. The downstream resolver needs both to recognise the rename and keep the old name as an alias.
|
|
17
26
|
|
|
18
27
|
## Output Format
|
|
19
28
|
|
|
20
|
-
Respond with ONLY a JSON
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
Rename example — note BOTH names are extracted:
|
|
32
|
-
Input: "Smara is now named Sigil"
|
|
33
|
-
Output:
|
|
34
|
-
[
|
|
35
|
-
{ "name": "smara", "description": "the project's previous name (renamed)" },
|
|
36
|
-
{ "name": "sigil", "description": "the project's current name; was previously called Smara" }
|
|
37
|
-
]
|
|
29
|
+
Respond with ONLY a JSON object with one key:
|
|
30
|
+
```json
|
|
31
|
+
{
|
|
32
|
+
"entities": [
|
|
33
|
+
{ "name": "user", "description": "the owner of this knowledge base" },
|
|
34
|
+
{ "name": "sigil", "description": "a local-first agent memory tool, previously named Smara" },
|
|
35
|
+
{ "name": "smara", "description": "the previous name of Sigil (renamed)" },
|
|
36
|
+
{ "name": "postgres", "description": "the database Sigil uses for durable storage" }
|
|
37
|
+
]
|
|
38
|
+
}
|
|
39
|
+
```
|
|
@@ -12,6 +12,20 @@ Extracting both together keeps them consistent: every relationship's endpoints s
|
|
|
12
12
|
- Do NOT extract generic terms ("programming", "software", "data").
|
|
13
13
|
- If two facts mention the same thing with different wording, list it once.
|
|
14
14
|
|
|
15
|
+
### The knowledge-base owner — the one exception to "no generic terms"
|
|
16
|
+
|
|
17
|
+
Many facts describe the person who owns this knowledge base, and they almost never name them. They are written in the third person ("User prefers concise responses", "User's name is Anmol") or the first person ("I use Postgres", "my editor is Neovim").
|
|
18
|
+
|
|
19
|
+
That person is a real entity — usually the most connected one in the graph — so:
|
|
20
|
+
|
|
21
|
+
- Refer to them with the exact name `user`, always lowercase, and list them once in `entities` with a description like "the owner of this knowledge base".
|
|
22
|
+
- Emit the relationship the fact asserts about them, with `user` as the subject:
|
|
23
|
+
`{"subject": "user", "relationship": "prefers", "object": "concise communication"}`.
|
|
24
|
+
- Use `prefers` / `dislikes` for opinions and taste, and ordinary verbs ("uses", "works on") for everything else.
|
|
25
|
+
- Do this even though the fact never spells out their name. Attaching the preference to `user` is the whole point — an object with no subject is an orphan.
|
|
26
|
+
|
|
27
|
+
`user` is the ONLY generic term you may extract. Every other rule below still applies to it.
|
|
28
|
+
|
|
15
29
|
### Rename contexts — ALWAYS extract BOTH names
|
|
16
30
|
|
|
17
31
|
When the source mentions a rename ("X is now named Y", "X was renamed to Y", "X used to be called Y") extract **both** the old and new names as separate entities, and also emit a "renamed from" relationship between them. The downstream resolver needs both names to recognise the rename and keep the old name as an alias.
|
|
@@ -30,13 +44,17 @@ Respond with ONLY a JSON object with two keys:
|
|
|
30
44
|
```json
|
|
31
45
|
{
|
|
32
46
|
"entities": [
|
|
47
|
+
{ "name": "user", "description": "the owner of this knowledge base" },
|
|
33
48
|
{ "name": "sigil", "description": "a local-first agent memory tool, previously named Smara" },
|
|
34
49
|
{ "name": "smara", "description": "the previous name of Sigil (renamed)" },
|
|
35
|
-
{ "name": "postgres", "description": "the database Sigil uses for durable storage" }
|
|
50
|
+
{ "name": "postgres", "description": "the database Sigil uses for durable storage" },
|
|
51
|
+
{ "name": "concise communication", "description": "short, direct replies without preamble" }
|
|
36
52
|
],
|
|
37
53
|
"relationships": [
|
|
38
54
|
{ "subject": "sigil", "relationship": "renamed from", "object": "smara" },
|
|
39
|
-
{ "subject": "sigil", "relationship": "uses", "object": "postgres" }
|
|
55
|
+
{ "subject": "sigil", "relationship": "uses", "object": "postgres" },
|
|
56
|
+
{ "subject": "user", "relationship": "works on", "object": "sigil" },
|
|
57
|
+
{ "subject": "user", "relationship": "prefers", "object": "concise communication" }
|
|
40
58
|
]
|
|
41
59
|
}
|
|
42
60
|
```
|
package/src/db/drivers/index.js
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* DB driver selection.
|
|
2
|
+
* DB driver selection — driven entirely by config.json (config.db.*); no env.
|
|
3
3
|
*
|
|
4
|
-
* -
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* - Else if
|
|
8
|
-
*
|
|
9
|
-
* -
|
|
10
|
-
*
|
|
4
|
+
* - config.db.mode === 'embedded' → the in-process PGlite engine (a full
|
|
5
|
+
* Postgres 17 + pgvector compiled to WASM, no server, no Docker). Data at
|
|
6
|
+
* ~/.sigil/db.
|
|
7
|
+
* - Else if config.db.url is set → the URL driver (Neon, Supabase, RDS,
|
|
8
|
+
* Render, Railway, self-hosted, etc.).
|
|
9
|
+
* - Else if config.db.mode === 'local' → the local-postgres driver from the
|
|
10
|
+
* stored host/port/name/user/password.
|
|
11
|
+
* - Otherwise → throw notConfiguredError() (setup never finished).
|
|
11
12
|
*
|
|
12
13
|
* Returns a descriptor: { kind, provider, connection, client }. `connection` is
|
|
13
14
|
* a pg-shape object for the URL/local drivers, or { pglitePath } for embedded.
|
|
@@ -22,7 +23,8 @@ import { buildUrlConnection, classifyProvider } from './url.js';
|
|
|
22
23
|
import { ClientPGlite, PGLITE_DB_PATH } from '../pglite-adapter.js';
|
|
23
24
|
|
|
24
25
|
export function selectDriver(config) {
|
|
25
|
-
|
|
26
|
+
const mode = config.db.mode;
|
|
27
|
+
if (mode === 'embedded') {
|
|
26
28
|
return {
|
|
27
29
|
kind: 'embedded',
|
|
28
30
|
provider: 'pglite',
|
|
@@ -39,12 +41,33 @@ export function selectDriver(config) {
|
|
|
39
41
|
client: 'pg',
|
|
40
42
|
};
|
|
41
43
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
44
|
+
// Field-based local Postgres — ONLY when the user EXPLICITLY chose it (setup
|
|
45
|
+
// persisted mode 'local'). A null/unknown mode with no URL means setup never
|
|
46
|
+
// finished — and building a local driver here would silently point config.db's
|
|
47
|
+
// defaults (localhost:5432, user sigil_app) at whatever foreign Postgres owns
|
|
48
|
+
// that port, producing a baffling auth error against someone else's database
|
|
49
|
+
// instead of an honest "Sigil isn't set up". Fail loud instead. (config.json
|
|
50
|
+
// is the sole source of truth — the legacy SIGIL_DB_* env escape hatch is gone.)
|
|
51
|
+
if (mode === 'local') {
|
|
52
|
+
return {
|
|
53
|
+
kind: 'local',
|
|
54
|
+
provider: 'local',
|
|
55
|
+
connection: normalizeConnection(buildLocalConnection(config)),
|
|
56
|
+
client: 'pg',
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
throw notConfiguredError();
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function notConfiguredError() {
|
|
63
|
+
const err = new Error(
|
|
64
|
+
'Sigil has no database configured yet. Run `sigil` to set up (the built-in '
|
|
65
|
+
+ 'zero-prerequisite embedded engine) or `sigil init` in the terminal. Refusing to '
|
|
66
|
+
+ 'connect to a default localhost:5432 — that would hit whatever Postgres happens to '
|
|
67
|
+
+ 'own that port, not Sigil\'s.',
|
|
68
|
+
);
|
|
69
|
+
err.code = 'not_configured';
|
|
70
|
+
return err;
|
|
48
71
|
}
|
|
49
72
|
|
|
50
73
|
// pg throws "SASL: client password must be a string" if password is null/
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Add caller attribution to trace_event.
|
|
3
|
+
*
|
|
4
|
+
* agent — which agent originated the traced operation: 'claude-code',
|
|
5
|
+
* 'codex', 'cursor', 'mcp', 'cli', etc. NULL = unknown /
|
|
6
|
+
* pre-migration. Sourced from the authenticated caller's
|
|
7
|
+
* request-context (AsyncLocalStorage) via currentAgent().
|
|
8
|
+
* session_id — the originating session (e.g. a Claude Code session id),
|
|
9
|
+
* passed explicitly by the caller (the read hook forwards it).
|
|
10
|
+
* NULL when the caller doesn't carry one (CLI, MCP).
|
|
11
|
+
*
|
|
12
|
+
* This is PROVENANCE for the Activity feed, mirroring fact.created_by_agent
|
|
13
|
+
* (20260601000005). It answers "which session of which agent made this
|
|
14
|
+
* search/expansion call" — previously the trace recorded only device_id +
|
|
15
|
+
* transport, so unprompted hook-driven searches were unattributable.
|
|
16
|
+
*/
|
|
17
|
+
exports.up = (knex) =>
|
|
18
|
+
knex.schema.alterTable('trace_event', (t) => {
|
|
19
|
+
t.text('agent');
|
|
20
|
+
t.text('session_id');
|
|
21
|
+
t.index(['agent', 'ts'], 'trace_event_agent_ts_idx');
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
exports.down = (knex) =>
|
|
25
|
+
knex.schema.alterTable('trace_event', (t) => {
|
|
26
|
+
t.dropIndex(['agent', 'ts'], 'trace_event_agent_ts_idx');
|
|
27
|
+
t.dropColumn('agent');
|
|
28
|
+
t.dropColumn('session_id');
|
|
29
|
+
});
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Add managed-session correlation to llm_log.
|
|
3
|
+
*
|
|
4
|
+
* worker_id — the warm worker that served this call (e.g. 'claude-0',
|
|
5
|
+
* whose tmux session is 'sigil-claude-0'). NULL for one-shot
|
|
6
|
+
* (claude -p) and API-provider calls.
|
|
7
|
+
* req_id — the reqId correlating an llm_log row to a managed-session
|
|
8
|
+
* dispatch/result event in trace_event (kind='engine').
|
|
9
|
+
* via_fallback — true when the managed-session engine bailed to the one-shot
|
|
10
|
+
* claude-cli path (no workers, dead-man timeout, boot circuit
|
|
11
|
+
* breaker). Lets you see warm-path hit rate at a glance.
|
|
12
|
+
*
|
|
13
|
+
* Before this, llm_log carried only `caller` (functional role) — there was no
|
|
14
|
+
* way to attribute a call to a specific warm worker or know whether it took
|
|
15
|
+
* the warm path. `caller` already exists from 20260405140000.
|
|
16
|
+
*/
|
|
17
|
+
exports.up = (knex) =>
|
|
18
|
+
knex.schema.alterTable('llm_log', (t) => {
|
|
19
|
+
t.text('worker_id');
|
|
20
|
+
t.text('req_id');
|
|
21
|
+
t.boolean('via_fallback');
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
exports.down = (knex) =>
|
|
25
|
+
knex.schema.alterTable('llm_log', (t) => {
|
|
26
|
+
t.dropColumn('worker_id');
|
|
27
|
+
t.dropColumn('req_id');
|
|
28
|
+
t.dropColumn('via_fallback');
|
|
29
|
+
});
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Store the full original text on the document row.
|
|
3
|
+
*
|
|
4
|
+
* Until now `document` held only metadata (source_path, title, content_hash,
|
|
5
|
+
* counts) and the text lived exclusively in `chunk`. That made "give me this
|
|
6
|
+
* document back, whole" impossible to answer exactly: chunks overlap by ~50
|
|
7
|
+
* tokens (src/ingestion/chunker.js), so concatenating them duplicates text at
|
|
8
|
+
* every seam, and re-reading source_path fails for pasted content and session
|
|
9
|
+
* histories (which get synthetic `raw/<ts>` / `thought:<hash>` paths) or any
|
|
10
|
+
* file that has since moved.
|
|
11
|
+
*
|
|
12
|
+
* Nullable on purpose: documents ingested before this migration have no stored
|
|
13
|
+
* content, and getDocument() falls back to de-overlapped chunk reassembly for
|
|
14
|
+
* them rather than failing. Re-ingesting a document fills the column in.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
exports.up = function (knex) {
|
|
18
|
+
return knex.schema.alterTable('document', (table) => {
|
|
19
|
+
table.text('content');
|
|
20
|
+
});
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
exports.down = function (knex) {
|
|
24
|
+
return knex.schema.alterTable('document', (table) => {
|
|
25
|
+
table.dropColumn('content');
|
|
26
|
+
});
|
|
27
|
+
};
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Allow llm_log.model to be NULL.
|
|
3
|
+
*
|
|
4
|
+
* The column was NOT NULL, which asserted something untrue: for providers that
|
|
5
|
+
* pick their own model (claude-cli, where config.llm.model is null and the CLI
|
|
6
|
+
* chooses), the effective model is only known from a SUCCESSFUL response. On
|
|
7
|
+
* the error path there is no response, so llm.js logs the *configured* model —
|
|
8
|
+
* null — and the insert died with:
|
|
9
|
+
*
|
|
10
|
+
* null value in column "model" of relation "llm_log" violates not-null
|
|
11
|
+
*
|
|
12
|
+
* The effect was the exact inverse of what the log is for: every successful
|
|
13
|
+
* call was recorded, and every FAILED call — the one you actually need to
|
|
14
|
+
* diagnose a broken provider — was silently dropped, leaving only a rate-limited
|
|
15
|
+
* "[llm-log] write failed" line that named the constraint rather than the outage.
|
|
16
|
+
*
|
|
17
|
+
* NULL now means "unknown model", which is the honest value for a failed call
|
|
18
|
+
* to a CLI-driven provider. The index is unaffected (Postgres indexes NULLs).
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
exports.up = function (knex) {
|
|
22
|
+
return knex.schema.alterTable('llm_log', (table) => {
|
|
23
|
+
table.text('model').nullable().alter();
|
|
24
|
+
});
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
exports.down = function (knex) {
|
|
28
|
+
// Backfill before restoring the constraint, or the ALTER fails on the very
|
|
29
|
+
// rows this migration made storable.
|
|
30
|
+
return knex('llm_log')
|
|
31
|
+
.whereNull('model')
|
|
32
|
+
.update({ model: 'unknown' })
|
|
33
|
+
.then(() => knex.schema.alterTable('llm_log', (table) => {
|
|
34
|
+
table.text('model').notNullable().alter();
|
|
35
|
+
}));
|
|
36
|
+
};
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Record HOW a relation was arrived at.
|
|
3
|
+
*
|
|
4
|
+
* Every edge in `relation` looked identical regardless of origin: the table
|
|
5
|
+
* carried mention_count but nothing separating "a model read this off a
|
|
6
|
+
* sentence" from "these two entities keep appearing in the same fact". That was
|
|
7
|
+
* survivable while one LLM pass was the only writer. It stops being survivable
|
|
8
|
+
* the moment a second, cheaper writer exists — derived edges could never be
|
|
9
|
+
* trust-weighted in ranking, re-derived after the algorithm improves, or
|
|
10
|
+
* removed without also deleting asserted ones.
|
|
11
|
+
*
|
|
12
|
+
* derived_by 'llm-extract' graph-extractor read it from fact text
|
|
13
|
+
* 'co-occurrence' derived from fact_entity + named in batch
|
|
14
|
+
* confidence 'high' | 'medium' | 'low' — the namer's own certainty
|
|
15
|
+
* weight the PMI that made the pair a candidate; null for asserted edges
|
|
16
|
+
*
|
|
17
|
+
* Existing rows backfill to 'llm-extract' because that is literally where they
|
|
18
|
+
* came from — graph-extractor was the only writer before this.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
exports.up = function (knex) {
|
|
22
|
+
return knex.schema
|
|
23
|
+
.alterTable('relation', (table) => {
|
|
24
|
+
table.text('derived_by').notNullable().defaultTo('llm-extract');
|
|
25
|
+
table.text('confidence');
|
|
26
|
+
table.float('weight');
|
|
27
|
+
})
|
|
28
|
+
.then(() => knex.schema.alterTable('relation', (table) => {
|
|
29
|
+
// Derived edges are queried and pruned as a set, so the discriminator
|
|
30
|
+
// earns an index rather than a sequential scan over every edge.
|
|
31
|
+
table.index('derived_by', 'relation_derived_by_idx');
|
|
32
|
+
}));
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
exports.down = function (knex) {
|
|
36
|
+
return knex.schema.alterTable('relation', (table) => {
|
|
37
|
+
table.dropIndex('derived_by', 'relation_derived_by_idx');
|
|
38
|
+
table.dropColumn('derived_by');
|
|
39
|
+
table.dropColumn('confidence');
|
|
40
|
+
table.dropColumn('weight');
|
|
41
|
+
});
|
|
42
|
+
};
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Add `visibility` to fact — which principals may retrieve it.
|
|
3
|
+
*
|
|
4
|
+
* Sigil already records WHO wrote a fact (created_by_device_id,
|
|
5
|
+
* created_by_agent) and has always treated that as provenance only, never as a
|
|
6
|
+
* retrieval scope. That default is right and stays: cross-agent sharing is the
|
|
7
|
+
* product. Claude must see what Cursor wrote, and the cloud agent's notes on a
|
|
8
|
+
* bug are worthless if the laptop can't read them.
|
|
9
|
+
*
|
|
10
|
+
* But a small class of facts is genuinely addressed TO one assistant rather
|
|
11
|
+
* than being true of the user's world. "Call me sir", "always answer in
|
|
12
|
+
* bullets", "you are my pair-programmer" — told to one agent, these describe a
|
|
13
|
+
* relationship with that agent, and replaying them into a different agent puts
|
|
14
|
+
* words in its mouth the user never asked for. Provenance cannot express this,
|
|
15
|
+
* because provenance is never filtered on.
|
|
16
|
+
*
|
|
17
|
+
* Values (one column; the principal is implied by the value):
|
|
18
|
+
*
|
|
19
|
+
* 'shared' DEFAULT — every device and every agent of this owner. The
|
|
20
|
+
* overwhelming majority of facts. Matches what every surveyed
|
|
21
|
+
* sync system does at the product layer: sync everything, and
|
|
22
|
+
* carve out narrow, deliberate exceptions.
|
|
23
|
+
* 'agent' only the agent that wrote it (matched on created_by_agent).
|
|
24
|
+
* The "call me sir" class.
|
|
25
|
+
* 'device' only the install that wrote it (matched on created_by_device_id).
|
|
26
|
+
* Machine-local truth: a checkout path, a local port, a GPU.
|
|
27
|
+
*
|
|
28
|
+
* Why a column and not derived from pod membership: the hook dispatcher makes
|
|
29
|
+
* the PROJECT pod own every hook write, so an agent-directed instruction typed
|
|
30
|
+
* inside a repo lands in a pod whose kind is `shared`. The pod says what the
|
|
31
|
+
* fact is ABOUT; it cannot say who it is FOR.
|
|
32
|
+
*
|
|
33
|
+
* NOT NULL with a default, so the migration is backfill-free and every
|
|
34
|
+
* pre-existing fact keeps its current, fully-visible behaviour. Changing a
|
|
35
|
+
* fact's visibility later is supported and takes effect on the next read —
|
|
36
|
+
* there is no cached copy on another device to chase, because in a shared
|
|
37
|
+
* database there is only ever one row.
|
|
38
|
+
*/
|
|
39
|
+
exports.up = async (knex) => {
|
|
40
|
+
await knex.schema.alterTable('fact', (t) => {
|
|
41
|
+
t.text('visibility').notNullable().defaultTo('shared');
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
// Reads filter on visibility on EVERY query, and the overwhelmingly common
|
|
45
|
+
// value is 'shared'. A partial index over just the exceptions keeps the
|
|
46
|
+
// index tiny (a few rows out of the whole store) while still letting the
|
|
47
|
+
// planner find them; the 'shared' majority is served by the existing
|
|
48
|
+
// namespace/status indexes without this one bloating them.
|
|
49
|
+
await knex.raw(`
|
|
50
|
+
CREATE INDEX idx_fact_visibility_scoped
|
|
51
|
+
ON fact (visibility, created_by_agent, created_by_device_id)
|
|
52
|
+
WHERE visibility <> 'shared'
|
|
53
|
+
`);
|
|
54
|
+
|
|
55
|
+
await knex.raw(`
|
|
56
|
+
ALTER TABLE fact ADD CONSTRAINT fact_visibility_check
|
|
57
|
+
CHECK (visibility IN ('shared', 'agent', 'device'))
|
|
58
|
+
`);
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
exports.down = async (knex) => {
|
|
62
|
+
await knex.raw('ALTER TABLE fact DROP CONSTRAINT IF EXISTS fact_visibility_check');
|
|
63
|
+
await knex.raw('DROP INDEX IF EXISTS idx_fact_visibility_scoped');
|
|
64
|
+
await knex.schema.alterTable('fact', (t) => {
|
|
65
|
+
t.dropColumn('visibility');
|
|
66
|
+
});
|
|
67
|
+
};
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Re-score facts that were stamped `vital` by default rather than by judgement.
|
|
3
|
+
*
|
|
4
|
+
* Three code paths defaulted importance UP to 'vital' when the classifier
|
|
5
|
+
* didn't return a usable value: the thought fast-path in ingestion/pipeline.js,
|
|
6
|
+
* the validation fallback in cognitive/input-classifier.js, and the atomic
|
|
7
|
+
* path used when classification failed outright. Measured on a real store,
|
|
8
|
+
* that produced 50 vital facts out of 62.
|
|
9
|
+
*
|
|
10
|
+
* At 81% saturation the flag stops being information. It is read in two places
|
|
11
|
+
* and degrades both:
|
|
12
|
+
*
|
|
13
|
+
* - the `vital` hot-context kind, which owns always-on slots in every
|
|
14
|
+
* prompt — so the slots filled with whatever had been ingested most
|
|
15
|
+
* recently, from any project;
|
|
16
|
+
* - IMPORTANCE_VITAL_MULT in the search ranking, which multiplies nearly
|
|
17
|
+
* every row by the same constant and therefore orders nothing.
|
|
18
|
+
*
|
|
19
|
+
* The three defaults are fixed at the source in this same change. This
|
|
20
|
+
* migration repairs the rows already written under them.
|
|
21
|
+
*
|
|
22
|
+
* WHAT IS AND ISN'T RECOVERABLE
|
|
23
|
+
*
|
|
24
|
+
* Nothing distinguishes "the model judged this vital" from "the model said
|
|
25
|
+
* nothing and the code filled in vital" — both wrote the identical string.
|
|
26
|
+
* That information is gone, so this is a heuristic, not a reconstruction.
|
|
27
|
+
*
|
|
28
|
+
* The heuristic: keep vital only where the category is one the user's standing
|
|
29
|
+
* context genuinely depends on (the same set the directive kind selects on),
|
|
30
|
+
* and demote the rest. This errs toward demotion because the two errors are
|
|
31
|
+
* not symmetric — a wrongly-demoted fact is still fully searchable and simply
|
|
32
|
+
* stops occupying a permanent slot, while a wrongly-kept one goes on crowding
|
|
33
|
+
* out the facts that belong there. Re-promote any individual fact by hand if
|
|
34
|
+
* it deserves it.
|
|
35
|
+
*
|
|
36
|
+
* `down` restores vital to exactly the rows this touched, which is why the
|
|
37
|
+
* predicate is repeated rather than blanket-restoring every supplementary row.
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
// Kept in sync with DIRECTIVE_CATEGORIES in memory/pods/kinds/directive.js.
|
|
41
|
+
// Duplicated deliberately: a migration must describe the world as it was when
|
|
42
|
+
// it ran, so it cannot import a constant that later changes underneath it.
|
|
43
|
+
const KEEP_VITAL = ['preference', 'opinion', 'personal', 'convention'];
|
|
44
|
+
|
|
45
|
+
exports.up = async (knex) => {
|
|
46
|
+
await knex('fact')
|
|
47
|
+
.where('importance', 'vital')
|
|
48
|
+
.whereNotIn('category', KEEP_VITAL)
|
|
49
|
+
.update({ importance: 'supplementary', importance_score: 2 });
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
exports.down = async (knex) => {
|
|
53
|
+
await knex('fact')
|
|
54
|
+
.where('importance', 'supplementary')
|
|
55
|
+
.whereNotIn('category', KEEP_VITAL)
|
|
56
|
+
.update({ importance: 'vital', importance_score: 5 });
|
|
57
|
+
};
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable ingestion work queue.
|
|
3
|
+
*
|
|
4
|
+
* Document bytes are staged in payload before an async caller receives an ack.
|
|
5
|
+
* The runner leases jobs, so a daemon crash turns an expired `running` row back
|
|
6
|
+
* into queued work instead of silently losing it. Entity and relation work use
|
|
7
|
+
* the same queue and therefore never hold up the searchable fact commit.
|
|
8
|
+
*/
|
|
9
|
+
exports.up = async function up(knex) {
|
|
10
|
+
await knex.schema.createTable('ingestion_job', (t) => {
|
|
11
|
+
t.bigIncrements('id').primary();
|
|
12
|
+
t.text('uid').notNullable().unique();
|
|
13
|
+
t.text('kind').notNullable();
|
|
14
|
+
t.text('status').notNullable().defaultTo('queued');
|
|
15
|
+
t.text('stage').notNullable().defaultTo('queued');
|
|
16
|
+
t.text('namespace');
|
|
17
|
+
t.bigInteger('document_id').references('id').inTable('document').onDelete('SET NULL');
|
|
18
|
+
t.text('dedupe_key');
|
|
19
|
+
t.integer('priority').notNullable().defaultTo(0);
|
|
20
|
+
t.jsonb('payload').notNullable().defaultTo('{}');
|
|
21
|
+
t.jsonb('result');
|
|
22
|
+
t.jsonb('timings').notNullable().defaultTo('{}');
|
|
23
|
+
t.integer('attempts').notNullable().defaultTo(0);
|
|
24
|
+
t.integer('max_attempts').notNullable().defaultTo(3);
|
|
25
|
+
t.timestamp('available_at', { useTz: true }).notNullable().defaultTo(knex.fn.now());
|
|
26
|
+
t.text('lease_owner');
|
|
27
|
+
t.timestamp('lease_expires_at', { useTz: true });
|
|
28
|
+
t.text('error');
|
|
29
|
+
t.timestamp('created_at', { useTz: true }).notNullable().defaultTo(knex.fn.now());
|
|
30
|
+
t.timestamp('updated_at', { useTz: true }).notNullable().defaultTo(knex.fn.now());
|
|
31
|
+
t.timestamp('started_at', { useTz: true });
|
|
32
|
+
t.timestamp('completed_at', { useTz: true });
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
await knex.schema.alterTable('ingestion_job', (t) => {
|
|
36
|
+
t.index(['status', 'available_at', 'priority'], 'ingestion_job_claim_idx');
|
|
37
|
+
t.index(['namespace', 'created_at'], 'ingestion_job_namespace_idx');
|
|
38
|
+
t.index(['document_id'], 'ingestion_job_document_idx');
|
|
39
|
+
});
|
|
40
|
+
await knex.raw(`
|
|
41
|
+
CREATE UNIQUE INDEX ingestion_job_active_dedupe_idx
|
|
42
|
+
ON ingestion_job (dedupe_key)
|
|
43
|
+
WHERE dedupe_key IS NOT NULL AND status IN ('queued', 'running')
|
|
44
|
+
`);
|
|
45
|
+
await knex.raw(`
|
|
46
|
+
ALTER TABLE ingestion_job
|
|
47
|
+
ADD CONSTRAINT ingestion_job_status_check
|
|
48
|
+
CHECK (status IN ('queued', 'running', 'completed', 'failed'))
|
|
49
|
+
`);
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
exports.down = (knex) => knex.schema.dropTableIfExists('ingestion_job');
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/** Time spent waiting for a managed worker before model execution began. */
|
|
2
|
+
exports.up = (knex) => knex.schema.alterTable('llm_log', (t) => {
|
|
3
|
+
t.integer('queued_ms');
|
|
4
|
+
});
|
|
5
|
+
|
|
6
|
+
exports.down = (knex) => knex.schema.alterTable('llm_log', (t) => {
|
|
7
|
+
t.dropColumn('queued_ms');
|
|
8
|
+
});
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cheap exact-content revalidation at commit time.
|
|
3
|
+
*
|
|
4
|
+
* Fact preparation deliberately runs outside the write transaction. Two
|
|
5
|
+
* concurrent callers can therefore prepare the same new fact before either
|
|
6
|
+
* commits. The apply phase probes md5(content)+namespace under the serialized
|
|
7
|
+
* commit lock; this partial expression index keeps that guard O(log n) without
|
|
8
|
+
* trying to btree-index arbitrarily long fact text.
|
|
9
|
+
*/
|
|
10
|
+
exports.up = (knex) => knex.raw(`
|
|
11
|
+
CREATE INDEX fact_active_content_hash_idx
|
|
12
|
+
ON fact (namespace, md5(content))
|
|
13
|
+
WHERE status = 'active'
|
|
14
|
+
`);
|
|
15
|
+
|
|
16
|
+
exports.down = (knex) => knex.raw('DROP INDEX IF EXISTS fact_active_content_hash_idx');
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Remember negative relation judgments at the evidence version that produced
|
|
3
|
+
* them. A rejected co-occurrence pair is reconsidered only after its shared
|
|
4
|
+
* active-fact count grows, preventing every ingest from paying to reject the
|
|
5
|
+
* same pair again while still allowing new evidence to change the verdict.
|
|
6
|
+
*/
|
|
7
|
+
exports.up = async function up(knex) {
|
|
8
|
+
await knex.schema.createTable('relation_candidate_judgment', (t) => {
|
|
9
|
+
t.bigIncrements('id').primary();
|
|
10
|
+
t.bigInteger('source_id').notNullable().references('id').inTable('entity').onDelete('CASCADE');
|
|
11
|
+
t.bigInteger('target_id').notNullable().references('id').inTable('entity').onDelete('CASCADE');
|
|
12
|
+
t.integer('shared_facts').notNullable();
|
|
13
|
+
t.text('decision').notNullable().defaultTo('rejected');
|
|
14
|
+
t.timestamp('created_at', { useTz: true }).notNullable().defaultTo(knex.fn.now());
|
|
15
|
+
t.timestamp('updated_at', { useTz: true }).notNullable().defaultTo(knex.fn.now());
|
|
16
|
+
t.unique(['source_id', 'target_id'], { indexName: 'relation_candidate_judgment_pair_uniq' });
|
|
17
|
+
});
|
|
18
|
+
await knex.raw(`
|
|
19
|
+
ALTER TABLE relation_candidate_judgment
|
|
20
|
+
ADD CONSTRAINT relation_candidate_judgment_order_check CHECK (source_id < target_id),
|
|
21
|
+
ADD CONSTRAINT relation_candidate_judgment_decision_check CHECK (decision IN ('rejected'))
|
|
22
|
+
`);
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
exports.down = (knex) => knex.schema.dropTableIfExists('relation_candidate_judgment');
|