@anmol-srv/sigil 0.18.2 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +37 -25
  2. package/dist/cli.js +1191 -790
  3. package/dist/daemon.js +664 -302
  4. package/dist/hooks/post-tool-use.js +1 -69
  5. package/dist/hooks/session-end.js +13 -236
  6. package/dist/hooks/stop.js +36 -246
  7. package/dist/hooks/user-prompt-submit.js +25 -288
  8. package/dist/mcp/worker-server.js +6 -0
  9. package/dist/npm-entry.js +10 -0
  10. package/dist/server.js +64 -56
  11. package/install.sh +207 -0
  12. package/integrations/hermes/README.md +30 -24
  13. package/integrations/hermes/plugin/README.md +1 -1
  14. package/integrations/hermes/plugin/__init__.py +68 -24
  15. package/knexfile.js +19 -22
  16. package/package.json +9 -28
  17. package/prompts/audm-decision.md +26 -1
  18. package/prompts/entity-extraction.md +32 -30
  19. package/prompts/graph-extraction.md +20 -2
  20. package/src/db/drivers/index.js +38 -15
  21. package/src/db/migrations/20260629120000_add-trace-event-attribution.cjs +29 -0
  22. package/src/db/migrations/20260629120001_add-llm-log-session-correlation.cjs +29 -0
  23. package/src/db/migrations/20260806120000_add-document-content.cjs +27 -0
  24. package/src/db/migrations/20260806120001_llm-log-model-nullable.cjs +36 -0
  25. package/src/db/migrations/20260811120000_add-relation-provenance.cjs +42 -0
  26. package/src/db/migrations/20260811130000_add-fact-visibility.cjs +67 -0
  27. package/src/db/migrations/20260813090000_rescore-defaulted-vital.cjs +57 -0
  28. package/src/db/migrations/20260814120000_create-ingestion-job.cjs +52 -0
  29. package/src/db/migrations/20260814120100_add-llm-log-queued-ms.cjs +8 -0
  30. package/src/db/migrations/20260814120200_add-active-fact-content-hash-index.cjs +16 -0
  31. package/src/db/migrations/20260814120300_create-relation-candidate-judgment.cjs +25 -0
  32. package/src/gui/web/app.css +851 -84
  33. package/src/gui/web/app.js +796 -383
  34. package/src/gui/web/cmdk.js +238 -0
  35. package/src/gui/web/components.js +64 -0
  36. package/src/gui/web/design/colors_and_type.css +6 -1
  37. package/src/gui/web/graph/GraphIsland.jsx +450 -0
  38. package/src/gui/web/graph/cluster.js +305 -0
  39. package/src/gui/web/graph/cluster.test.js +219 -0
  40. package/src/gui/web/health.js +117 -0
  41. package/src/gui/web/health.test.js +159 -0
  42. package/src/gui/web/icons.js +65 -0
  43. package/src/gui/web/index.html +310 -115
  44. package/src/gui/web/settings.js +386 -0
  45. package/src/gui/web/setup.js +2 -2
  46. package/src/gui/web/toast.js +6 -3
  47. package/src/gui/web/vendor/graph-island.js +52 -0
@@ -1,37 +1,39 @@
1
- You are extracting topic entities from a set of facts in an organizational knowledge base.
2
- A "topic" is a distinct concept, technology, system, process, or subject referenced in the facts.
1
+ You are extracting the entities from a set of facts in a personal knowledge base: the distinct topics, concepts, technologies, systems, products, people, or named things the facts are about.
3
2
 
4
- ## Rules
3
+ Extract entities ONLY. Do not infer relationships between them — that is derived separately, from which entities actually co-occur across the whole store.
5
4
 
6
- - Extract 3-8 topics. Only meaningful, distinct topics.
7
- - Use canonical names: "normalization" not "database normalization concepts", "React hooks" not "React.js hooks pattern".
8
- - If two facts mention the same topic with different wording, extract it once with the canonical name.
9
- - Include a brief description (1 sentence) for context.
10
- - Do NOT extract generic terms like "programming", "coding", "software". Be specific.
11
- - Do NOT extract people names or document titles — those are handled separately.
12
- - Topics should be reusable across documents — "database indexing" not "the indexing discussion in doc 12".
5
+ ## Entities
13
6
 
14
- ## Rename contexts — ALWAYS extract BOTH names
7
+ - Extract 3-8 meaningful, distinct entities. Use canonical names, lowercase: "react hooks" not "the React.js hooks pattern".
8
+ - Include a one-sentence description giving context.
9
+ - Do NOT extract generic terms ("programming", "software", "data").
10
+ - If two facts mention the same thing with different wording, list it once.
15
11
 
16
- When the source mentions a rename ("X is now named Y", "X has been renamed to Y", "X used to be called Y", "we renamed X to Y", etc.) — extract **both** the old and new names as separate topic entries. Do not collapse the rename into a single topic. The downstream entity resolver needs both names so it can recognise the rename and merge them into one entity with the old name preserved as an alias. Skipping the old name will cause the system to create a duplicate entity.
12
+ ### The knowledge-base owner — the one exception to "no generic terms"
13
+
14
+ Many facts describe the person who owns this knowledge base, and they almost never name them. They are written in the third person ("User prefers concise responses", "User's name is Anmol") or the first person ("I use Postgres", "my editor is Neovim").
15
+
16
+ That person is a real entity — usually the most connected one in the graph — so:
17
+
18
+ - Refer to them with the exact name `user`, always lowercase, and list them once with a description like "the owner of this knowledge base".
19
+ - Do this even though the fact never spells out their name.
20
+
21
+ `user` is the ONLY generic term you may extract.
22
+
23
+ ### Rename contexts — ALWAYS extract BOTH names
24
+
25
+ When the source mentions a rename ("X is now named Y", "X was renamed to Y", "X used to be called Y") extract **both** the old and new names as separate entities. The downstream resolver needs both to recognise the rename and keep the old name as an alias.
17
26
 
18
27
  ## Output Format
19
28
 
20
- Respond with ONLY a JSON array. Each item:
21
- - "name" (string): canonical topic name, lowercase
22
- - "description" (string): one-sentence description of what this topic covers in context
23
-
24
- Example:
25
- [
26
- { "name": "3NF normalization", "description": "Third normal form and eliminating transitive dependencies in relational databases" },
27
- { "name": "foreign key cascades", "description": "CASCADE vs SET NULL behavior when deleting referenced rows" },
28
- { "name": "query optimization", "description": "Techniques for improving SQL query performance including indexing and query plans" }
29
- ]
30
-
31
- Rename example — note BOTH names are extracted:
32
- Input: "Smara is now named Sigil"
33
- Output:
34
- [
35
- { "name": "smara", "description": "the project's previous name (renamed)" },
36
- { "name": "sigil", "description": "the project's current name; was previously called Smara" }
37
- ]
29
+ Respond with ONLY a JSON object with one key:
30
+ ```json
31
+ {
32
+ "entities": [
33
+ { "name": "user", "description": "the owner of this knowledge base" },
34
+ { "name": "sigil", "description": "a local-first agent memory tool, previously named Smara" },
35
+ { "name": "smara", "description": "the previous name of Sigil (renamed)" },
36
+ { "name": "postgres", "description": "the database Sigil uses for durable storage" }
37
+ ]
38
+ }
39
+ ```
@@ -12,6 +12,20 @@ Extracting both together keeps them consistent: every relationship's endpoints s
12
12
  - Do NOT extract generic terms ("programming", "software", "data").
13
13
  - If two facts mention the same thing with different wording, list it once.
14
14
 
15
+ ### The knowledge-base owner — the one exception to "no generic terms"
16
+
17
+ Many facts describe the person who owns this knowledge base, and they almost never name them. They are written in the third person ("User prefers concise responses", "User's name is Anmol") or the first person ("I use Postgres", "my editor is Neovim").
18
+
19
+ That person is a real entity — usually the most connected one in the graph — so:
20
+
21
+ - Refer to them with the exact name `user`, always lowercase, and list them once in `entities` with a description like "the owner of this knowledge base".
22
+ - Emit the relationship the fact asserts about them, with `user` as the subject:
23
+ `{"subject": "user", "relationship": "prefers", "object": "concise communication"}`.
24
+ - Use `prefers` / `dislikes` for opinions and taste, and ordinary verbs ("uses", "works on") for everything else.
25
+ - Do this even though the fact never spells out their name. Attaching the preference to `user` is the whole point — an object with no subject is an orphan.
26
+
27
+ `user` is the ONLY generic term you may extract. Every other rule below still applies to it.
28
+
15
29
  ### Rename contexts — ALWAYS extract BOTH names
16
30
 
17
31
  When the source mentions a rename ("X is now named Y", "X was renamed to Y", "X used to be called Y") extract **both** the old and new names as separate entities, and also emit a "renamed from" relationship between them. The downstream resolver needs both names to recognise the rename and keep the old name as an alias.
@@ -30,13 +44,17 @@ Respond with ONLY a JSON object with two keys:
30
44
  ```json
31
45
  {
32
46
  "entities": [
47
+ { "name": "user", "description": "the owner of this knowledge base" },
33
48
  { "name": "sigil", "description": "a local-first agent memory tool, previously named Smara" },
34
49
  { "name": "smara", "description": "the previous name of Sigil (renamed)" },
35
- { "name": "postgres", "description": "the database Sigil uses for durable storage" }
50
+ { "name": "postgres", "description": "the database Sigil uses for durable storage" },
51
+ { "name": "concise communication", "description": "short, direct replies without preamble" }
36
52
  ],
37
53
  "relationships": [
38
54
  { "subject": "sigil", "relationship": "renamed from", "object": "smara" },
39
- { "subject": "sigil", "relationship": "uses", "object": "postgres" }
55
+ { "subject": "sigil", "relationship": "uses", "object": "postgres" },
56
+ { "subject": "user", "relationship": "works on", "object": "sigil" },
57
+ { "subject": "user", "relationship": "prefers", "object": "concise communication" }
40
58
  ]
41
59
  }
42
60
  ```
@@ -1,13 +1,14 @@
1
1
  /**
2
- * DB driver selection.
2
+ * DB driver selection — driven entirely by config.json (config.db.*); no env.
3
3
  *
4
- * - If db.mode === 'embedded' (or SIGIL_DB_MODE=embedded), use the in-process
5
- * PGlite engine — a full Postgres 17 + pgvector compiled to WASM, no server,
6
- * no Docker, no prerequisites. Data lives at ~/.sigil/db.
7
- * - Else if SIGIL_DATABASE_URL or DATABASE_URL is set, use the URL driver
8
- * (Neon, Supabase, RDS, Render, Railway, self-hosted, etc.).
9
- * - Otherwise, use the local-postgres driver (back-compat with the
10
- * SIGIL_DB_HOST / PORT / NAME / USER / PASSWORD env vars).
4
+ * - config.db.mode === 'embedded' → the in-process PGlite engine (a full
5
+ * Postgres 17 + pgvector compiled to WASM, no server, no Docker). Data at
6
+ * ~/.sigil/db.
7
+ * - Else if config.db.url is set → the URL driver (Neon, Supabase, RDS,
8
+ * Render, Railway, self-hosted, etc.).
9
+ * - Else if config.db.mode === 'local' → the local-postgres driver from the
10
+ * stored host/port/name/user/password.
11
+ * - Otherwise → throw notConfiguredError() (setup never finished).
11
12
  *
12
13
  * Returns a descriptor: { kind, provider, connection, client }. `connection` is
13
14
  * a pg-shape object for the URL/local drivers, or { pglitePath } for embedded.
@@ -22,7 +23,8 @@ import { buildUrlConnection, classifyProvider } from './url.js';
22
23
  import { ClientPGlite, PGLITE_DB_PATH } from '../pglite-adapter.js';
23
24
 
24
25
  export function selectDriver(config) {
25
- if (config.db.mode === 'embedded') {
26
+ const mode = config.db.mode;
27
+ if (mode === 'embedded') {
26
28
  return {
27
29
  kind: 'embedded',
28
30
  provider: 'pglite',
@@ -39,12 +41,33 @@ export function selectDriver(config) {
39
41
  client: 'pg',
40
42
  };
41
43
  }
42
- return {
43
- kind: 'local',
44
- provider: 'local',
45
- connection: normalizeConnection(buildLocalConnection(config)),
46
- client: 'pg',
47
- };
44
+ // Field-based local Postgres — ONLY when the user EXPLICITLY chose it (setup
45
+ // persisted mode 'local'). A null/unknown mode with no URL means setup never
46
+ // finished — and building a local driver here would silently point config.db's
47
+ // defaults (localhost:5432, user sigil_app) at whatever foreign Postgres owns
48
+ // that port, producing a baffling auth error against someone else's database
49
+ // instead of an honest "Sigil isn't set up". Fail loud instead. (config.json
50
+ // is the sole source of truth — the legacy SIGIL_DB_* env escape hatch is gone.)
51
+ if (mode === 'local') {
52
+ return {
53
+ kind: 'local',
54
+ provider: 'local',
55
+ connection: normalizeConnection(buildLocalConnection(config)),
56
+ client: 'pg',
57
+ };
58
+ }
59
+ throw notConfiguredError();
60
+ }
61
+
62
+ function notConfiguredError() {
63
+ const err = new Error(
64
+ 'Sigil has no database configured yet. Run `sigil` to set up (the built-in '
65
+ + 'zero-prerequisite embedded engine) or `sigil init` in the terminal. Refusing to '
66
+ + 'connect to a default localhost:5432 — that would hit whatever Postgres happens to '
67
+ + 'own that port, not Sigil\'s.',
68
+ );
69
+ err.code = 'not_configured';
70
+ return err;
48
71
  }
49
72
 
50
73
  // pg throws "SASL: client password must be a string" if password is null/
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Add caller attribution to trace_event.
3
+ *
4
+ * agent — which agent originated the traced operation: 'claude-code',
5
+ * 'codex', 'cursor', 'mcp', 'cli', etc. NULL = unknown /
6
+ * pre-migration. Sourced from the authenticated caller's
7
+ * request-context (AsyncLocalStorage) via currentAgent().
8
+ * session_id — the originating session (e.g. a Claude Code session id),
9
+ * passed explicitly by the caller (the read hook forwards it).
10
+ * NULL when the caller doesn't carry one (CLI, MCP).
11
+ *
12
+ * This is PROVENANCE for the Activity feed, mirroring fact.created_by_agent
13
+ * (20260601000005). It answers "which session of which agent made this
14
+ * search/expansion call" — previously the trace recorded only device_id +
15
+ * transport, so unprompted hook-driven searches were unattributable.
16
+ */
17
+ exports.up = (knex) =>
18
+ knex.schema.alterTable('trace_event', (t) => {
19
+ t.text('agent');
20
+ t.text('session_id');
21
+ t.index(['agent', 'ts'], 'trace_event_agent_ts_idx');
22
+ });
23
+
24
+ exports.down = (knex) =>
25
+ knex.schema.alterTable('trace_event', (t) => {
26
+ t.dropIndex(['agent', 'ts'], 'trace_event_agent_ts_idx');
27
+ t.dropColumn('agent');
28
+ t.dropColumn('session_id');
29
+ });
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Add managed-session correlation to llm_log.
3
+ *
4
+ * worker_id — the warm worker that served this call (e.g. 'claude-0',
5
+ * whose tmux session is 'sigil-claude-0'). NULL for one-shot
6
+ * (claude -p) and API-provider calls.
7
+ * req_id — the reqId correlating an llm_log row to a managed-session
8
+ * dispatch/result event in trace_event (kind='engine').
9
+ * via_fallback — true when the managed-session engine bailed to the one-shot
10
+ * claude-cli path (no workers, dead-man timeout, boot circuit
11
+ * breaker). Lets you see warm-path hit rate at a glance.
12
+ *
13
+ * Before this, llm_log carried only `caller` (functional role) — there was no
14
+ * way to attribute a call to a specific warm worker or know whether it took
15
+ * the warm path. `caller` already exists from 20260405140000.
16
+ */
17
+ exports.up = (knex) =>
18
+ knex.schema.alterTable('llm_log', (t) => {
19
+ t.text('worker_id');
20
+ t.text('req_id');
21
+ t.boolean('via_fallback');
22
+ });
23
+
24
+ exports.down = (knex) =>
25
+ knex.schema.alterTable('llm_log', (t) => {
26
+ t.dropColumn('worker_id');
27
+ t.dropColumn('req_id');
28
+ t.dropColumn('via_fallback');
29
+ });
@@ -0,0 +1,27 @@
1
+ /**
2
+ * Store the full original text on the document row.
3
+ *
4
+ * Until now `document` held only metadata (source_path, title, content_hash,
5
+ * counts) and the text lived exclusively in `chunk`. That made "give me this
6
+ * document back, whole" impossible to answer exactly: chunks overlap by ~50
7
+ * tokens (src/ingestion/chunker.js), so concatenating them duplicates text at
8
+ * every seam, and re-reading source_path fails for pasted content and session
9
+ * histories (which get synthetic `raw/<ts>` / `thought:<hash>` paths) or any
10
+ * file that has since moved.
11
+ *
12
+ * Nullable on purpose: documents ingested before this migration have no stored
13
+ * content, and getDocument() falls back to de-overlapped chunk reassembly for
14
+ * them rather than failing. Re-ingesting a document fills the column in.
15
+ */
16
+
17
+ exports.up = function (knex) {
18
+ return knex.schema.alterTable('document', (table) => {
19
+ table.text('content');
20
+ });
21
+ };
22
+
23
+ exports.down = function (knex) {
24
+ return knex.schema.alterTable('document', (table) => {
25
+ table.dropColumn('content');
26
+ });
27
+ };
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Allow llm_log.model to be NULL.
3
+ *
4
+ * The column was NOT NULL, which asserted something untrue: for providers that
5
+ * pick their own model (claude-cli, where config.llm.model is null and the CLI
6
+ * chooses), the effective model is only known from a SUCCESSFUL response. On
7
+ * the error path there is no response, so llm.js logs the *configured* model —
8
+ * null — and the insert died with:
9
+ *
10
+ * null value in column "model" of relation "llm_log" violates not-null
11
+ *
12
+ * The effect was the exact inverse of what the log is for: every successful
13
+ * call was recorded, and every FAILED call — the one you actually need to
14
+ * diagnose a broken provider — was silently dropped, leaving only a rate-limited
15
+ * "[llm-log] write failed" line that named the constraint rather than the outage.
16
+ *
17
+ * NULL now means "unknown model", which is the honest value for a failed call
18
+ * to a CLI-driven provider. The index is unaffected (Postgres indexes NULLs).
19
+ */
20
+
21
+ exports.up = function (knex) {
22
+ return knex.schema.alterTable('llm_log', (table) => {
23
+ table.text('model').nullable().alter();
24
+ });
25
+ };
26
+
27
+ exports.down = function (knex) {
28
+ // Backfill before restoring the constraint, or the ALTER fails on the very
29
+ // rows this migration made storable.
30
+ return knex('llm_log')
31
+ .whereNull('model')
32
+ .update({ model: 'unknown' })
33
+ .then(() => knex.schema.alterTable('llm_log', (table) => {
34
+ table.text('model').notNullable().alter();
35
+ }));
36
+ };
@@ -0,0 +1,42 @@
1
+ /**
2
+ * Record HOW a relation was arrived at.
3
+ *
4
+ * Every edge in `relation` looked identical regardless of origin: the table
5
+ * carried mention_count but nothing separating "a model read this off a
6
+ * sentence" from "these two entities keep appearing in the same fact". That was
7
+ * survivable while one LLM pass was the only writer. It stops being survivable
8
+ * the moment a second, cheaper writer exists — derived edges could never be
9
+ * trust-weighted in ranking, re-derived after the algorithm improves, or
10
+ * removed without also deleting asserted ones.
11
+ *
12
+ * derived_by 'llm-extract' graph-extractor read it from fact text
13
+ * 'co-occurrence' derived from fact_entity + named in batch
14
+ * confidence 'high' | 'medium' | 'low' — the namer's own certainty
15
+ * weight the PMI that made the pair a candidate; null for asserted edges
16
+ *
17
+ * Existing rows backfill to 'llm-extract' because that is literally where they
18
+ * came from — graph-extractor was the only writer before this.
19
+ */
20
+
21
+ exports.up = function (knex) {
22
+ return knex.schema
23
+ .alterTable('relation', (table) => {
24
+ table.text('derived_by').notNullable().defaultTo('llm-extract');
25
+ table.text('confidence');
26
+ table.float('weight');
27
+ })
28
+ .then(() => knex.schema.alterTable('relation', (table) => {
29
+ // Derived edges are queried and pruned as a set, so the discriminator
30
+ // earns an index rather than a sequential scan over every edge.
31
+ table.index('derived_by', 'relation_derived_by_idx');
32
+ }));
33
+ };
34
+
35
+ exports.down = function (knex) {
36
+ return knex.schema.alterTable('relation', (table) => {
37
+ table.dropIndex('derived_by', 'relation_derived_by_idx');
38
+ table.dropColumn('derived_by');
39
+ table.dropColumn('confidence');
40
+ table.dropColumn('weight');
41
+ });
42
+ };
@@ -0,0 +1,67 @@
1
+ /**
2
+ * Add `visibility` to fact — which principals may retrieve it.
3
+ *
4
+ * Sigil already records WHO wrote a fact (created_by_device_id,
5
+ * created_by_agent) and has always treated that as provenance only, never as a
6
+ * retrieval scope. That default is right and stays: cross-agent sharing is the
7
+ * product. Claude must see what Cursor wrote, and the cloud agent's notes on a
8
+ * bug are worthless if the laptop can't read them.
9
+ *
10
+ * But a small class of facts is genuinely addressed TO one assistant rather
11
+ * than being true of the user's world. "Call me sir", "always answer in
12
+ * bullets", "you are my pair-programmer" — told to one agent, these describe a
13
+ * relationship with that agent, and replaying them into a different agent puts
14
+ * words in its mouth the user never asked for. Provenance cannot express this,
15
+ * because provenance is never filtered on.
16
+ *
17
+ * Values (one column; the principal is implied by the value):
18
+ *
19
+ * 'shared' DEFAULT — every device and every agent of this owner. The
20
+ * overwhelming majority of facts. Matches what every surveyed
21
+ * sync system does at the product layer: sync everything, and
22
+ * carve out narrow, deliberate exceptions.
23
+ * 'agent' only the agent that wrote it (matched on created_by_agent).
24
+ * The "call me sir" class.
25
+ * 'device' only the install that wrote it (matched on created_by_device_id).
26
+ * Machine-local truth: a checkout path, a local port, a GPU.
27
+ *
28
+ * Why a column and not derived from pod membership: the hook dispatcher makes
29
+ * the PROJECT pod own every hook write, so an agent-directed instruction typed
30
+ * inside a repo lands in a pod whose kind is `shared`. The pod says what the
31
+ * fact is ABOUT; it cannot say who it is FOR.
32
+ *
33
+ * NOT NULL with a default, so the migration is backfill-free and every
34
+ * pre-existing fact keeps its current, fully-visible behaviour. Changing a
35
+ * fact's visibility later is supported and takes effect on the next read —
36
+ * there is no cached copy on another device to chase, because in a shared
37
+ * database there is only ever one row.
38
+ */
39
+ exports.up = async (knex) => {
40
+ await knex.schema.alterTable('fact', (t) => {
41
+ t.text('visibility').notNullable().defaultTo('shared');
42
+ });
43
+
44
+ // Reads filter on visibility on EVERY query, and the overwhelmingly common
45
+ // value is 'shared'. A partial index over just the exceptions keeps the
46
+ // index tiny (a few rows out of the whole store) while still letting the
47
+ // planner find them; the 'shared' majority is served by the existing
48
+ // namespace/status indexes without this one bloating them.
49
+ await knex.raw(`
50
+ CREATE INDEX idx_fact_visibility_scoped
51
+ ON fact (visibility, created_by_agent, created_by_device_id)
52
+ WHERE visibility <> 'shared'
53
+ `);
54
+
55
+ await knex.raw(`
56
+ ALTER TABLE fact ADD CONSTRAINT fact_visibility_check
57
+ CHECK (visibility IN ('shared', 'agent', 'device'))
58
+ `);
59
+ };
60
+
61
+ exports.down = async (knex) => {
62
+ await knex.raw('ALTER TABLE fact DROP CONSTRAINT IF EXISTS fact_visibility_check');
63
+ await knex.raw('DROP INDEX IF EXISTS idx_fact_visibility_scoped');
64
+ await knex.schema.alterTable('fact', (t) => {
65
+ t.dropColumn('visibility');
66
+ });
67
+ };
@@ -0,0 +1,57 @@
1
+ /**
2
+ * Re-score facts that were stamped `vital` by default rather than by judgement.
3
+ *
4
+ * Three code paths defaulted importance UP to 'vital' when the classifier
5
+ * didn't return a usable value: the thought fast-path in ingestion/pipeline.js,
6
+ * the validation fallback in cognitive/input-classifier.js, and the atomic
7
+ * path used when classification failed outright. Measured on a real store,
8
+ * that produced 50 vital facts out of 62.
9
+ *
10
+ * At 81% saturation the flag stops being information. It is read in two places
11
+ * and degrades both:
12
+ *
13
+ * - the `vital` hot-context kind, which owns always-on slots in every
14
+ * prompt — so the slots filled with whatever had been ingested most
15
+ * recently, from any project;
16
+ * - IMPORTANCE_VITAL_MULT in the search ranking, which multiplies nearly
17
+ * every row by the same constant and therefore orders nothing.
18
+ *
19
+ * The three defaults are fixed at the source in this same change. This
20
+ * migration repairs the rows already written under them.
21
+ *
22
+ * WHAT IS AND ISN'T RECOVERABLE
23
+ *
24
+ * Nothing distinguishes "the model judged this vital" from "the model said
25
+ * nothing and the code filled in vital" — both wrote the identical string.
26
+ * That information is gone, so this is a heuristic, not a reconstruction.
27
+ *
28
+ * The heuristic: keep vital only where the category is one the user's standing
29
+ * context genuinely depends on (the same set the directive kind selects on),
30
+ * and demote the rest. This errs toward demotion because the two errors are
31
+ * not symmetric — a wrongly-demoted fact is still fully searchable and simply
32
+ * stops occupying a permanent slot, while a wrongly-kept one goes on crowding
33
+ * out the facts that belong there. Re-promote any individual fact by hand if
34
+ * it deserves it.
35
+ *
36
+ * `down` restores vital to exactly the rows this touched, which is why the
37
+ * predicate is repeated rather than blanket-restoring every supplementary row.
38
+ */
39
+
40
+ // Kept in sync with DIRECTIVE_CATEGORIES in memory/pods/kinds/directive.js.
41
+ // Duplicated deliberately: a migration must describe the world as it was when
42
+ // it ran, so it cannot import a constant that later changes underneath it.
43
+ const KEEP_VITAL = ['preference', 'opinion', 'personal', 'convention'];
44
+
45
+ exports.up = async (knex) => {
46
+ await knex('fact')
47
+ .where('importance', 'vital')
48
+ .whereNotIn('category', KEEP_VITAL)
49
+ .update({ importance: 'supplementary', importance_score: 2 });
50
+ };
51
+
52
+ exports.down = async (knex) => {
53
+ await knex('fact')
54
+ .where('importance', 'supplementary')
55
+ .whereNotIn('category', KEEP_VITAL)
56
+ .update({ importance: 'vital', importance_score: 5 });
57
+ };
@@ -0,0 +1,52 @@
1
+ /**
2
+ * Durable ingestion work queue.
3
+ *
4
+ * Document bytes are staged in payload before an async caller receives an ack.
5
+ * The runner leases jobs, so a daemon crash turns an expired `running` row back
6
+ * into queued work instead of silently losing it. Entity and relation work use
7
+ * the same queue and therefore never hold up the searchable fact commit.
8
+ */
9
+ exports.up = async function up(knex) {
10
+ await knex.schema.createTable('ingestion_job', (t) => {
11
+ t.bigIncrements('id').primary();
12
+ t.text('uid').notNullable().unique();
13
+ t.text('kind').notNullable();
14
+ t.text('status').notNullable().defaultTo('queued');
15
+ t.text('stage').notNullable().defaultTo('queued');
16
+ t.text('namespace');
17
+ t.bigInteger('document_id').references('id').inTable('document').onDelete('SET NULL');
18
+ t.text('dedupe_key');
19
+ t.integer('priority').notNullable().defaultTo(0);
20
+ t.jsonb('payload').notNullable().defaultTo('{}');
21
+ t.jsonb('result');
22
+ t.jsonb('timings').notNullable().defaultTo('{}');
23
+ t.integer('attempts').notNullable().defaultTo(0);
24
+ t.integer('max_attempts').notNullable().defaultTo(3);
25
+ t.timestamp('available_at', { useTz: true }).notNullable().defaultTo(knex.fn.now());
26
+ t.text('lease_owner');
27
+ t.timestamp('lease_expires_at', { useTz: true });
28
+ t.text('error');
29
+ t.timestamp('created_at', { useTz: true }).notNullable().defaultTo(knex.fn.now());
30
+ t.timestamp('updated_at', { useTz: true }).notNullable().defaultTo(knex.fn.now());
31
+ t.timestamp('started_at', { useTz: true });
32
+ t.timestamp('completed_at', { useTz: true });
33
+ });
34
+
35
+ await knex.schema.alterTable('ingestion_job', (t) => {
36
+ t.index(['status', 'available_at', 'priority'], 'ingestion_job_claim_idx');
37
+ t.index(['namespace', 'created_at'], 'ingestion_job_namespace_idx');
38
+ t.index(['document_id'], 'ingestion_job_document_idx');
39
+ });
40
+ await knex.raw(`
41
+ CREATE UNIQUE INDEX ingestion_job_active_dedupe_idx
42
+ ON ingestion_job (dedupe_key)
43
+ WHERE dedupe_key IS NOT NULL AND status IN ('queued', 'running')
44
+ `);
45
+ await knex.raw(`
46
+ ALTER TABLE ingestion_job
47
+ ADD CONSTRAINT ingestion_job_status_check
48
+ CHECK (status IN ('queued', 'running', 'completed', 'failed'))
49
+ `);
50
+ };
51
+
52
+ exports.down = (knex) => knex.schema.dropTableIfExists('ingestion_job');
@@ -0,0 +1,8 @@
1
+ /** Time spent waiting for a managed worker before model execution began. */
2
+ exports.up = (knex) => knex.schema.alterTable('llm_log', (t) => {
3
+ t.integer('queued_ms');
4
+ });
5
+
6
+ exports.down = (knex) => knex.schema.alterTable('llm_log', (t) => {
7
+ t.dropColumn('queued_ms');
8
+ });
@@ -0,0 +1,16 @@
1
+ /**
2
+ * Cheap exact-content revalidation at commit time.
3
+ *
4
+ * Fact preparation deliberately runs outside the write transaction. Two
5
+ * concurrent callers can therefore prepare the same new fact before either
6
+ * commits. The apply phase probes md5(content)+namespace under the serialized
7
+ * commit lock; this partial expression index keeps that guard O(log n) without
8
+ * trying to btree-index arbitrarily long fact text.
9
+ */
10
+ exports.up = (knex) => knex.raw(`
11
+ CREATE INDEX fact_active_content_hash_idx
12
+ ON fact (namespace, md5(content))
13
+ WHERE status = 'active'
14
+ `);
15
+
16
+ exports.down = (knex) => knex.raw('DROP INDEX IF EXISTS fact_active_content_hash_idx');
@@ -0,0 +1,25 @@
1
+ /**
2
+ * Remember negative relation judgments at the evidence version that produced
3
+ * them. A rejected co-occurrence pair is reconsidered only after its shared
4
+ * active-fact count grows, preventing every ingest from paying to reject the
5
+ * same pair again while still allowing new evidence to change the verdict.
6
+ */
7
+ exports.up = async function up(knex) {
8
+ await knex.schema.createTable('relation_candidate_judgment', (t) => {
9
+ t.bigIncrements('id').primary();
10
+ t.bigInteger('source_id').notNullable().references('id').inTable('entity').onDelete('CASCADE');
11
+ t.bigInteger('target_id').notNullable().references('id').inTable('entity').onDelete('CASCADE');
12
+ t.integer('shared_facts').notNullable();
13
+ t.text('decision').notNullable().defaultTo('rejected');
14
+ t.timestamp('created_at', { useTz: true }).notNullable().defaultTo(knex.fn.now());
15
+ t.timestamp('updated_at', { useTz: true }).notNullable().defaultTo(knex.fn.now());
16
+ t.unique(['source_id', 'target_id'], { indexName: 'relation_candidate_judgment_pair_uniq' });
17
+ });
18
+ await knex.raw(`
19
+ ALTER TABLE relation_candidate_judgment
20
+ ADD CONSTRAINT relation_candidate_judgment_order_check CHECK (source_id < target_id),
21
+ ADD CONSTRAINT relation_candidate_judgment_decision_check CHECK (decision IN ('rejected'))
22
+ `);
23
+ };
24
+
25
+ exports.down = (knex) => knex.schema.dropTableIfExists('relation_candidate_judgment');