latticesql 4.2.4 → 4.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -1
- package/dist/cli.js +5923 -873
- package/dist/desktop-entry.js +6036 -858
- package/dist/gui-assets/ort/ort-wasm-simd-threaded.asyncify.mjs +112 -0
- package/dist/gui-assets/ort/ort-wasm-simd-threaded.asyncify.wasm +0 -0
- package/dist/gui-assets/ort/ort-wasm-simd-threaded.jsep.mjs +106 -0
- package/dist/gui-assets/ort/ort-wasm-simd-threaded.jsep.wasm +0 -0
- package/dist/gui-assets/ort/ort-wasm-simd-threaded.mjs +59 -0
- package/dist/gui-assets/ort/ort-wasm-simd-threaded.wasm +0 -0
- package/dist/gui-assets/transcriber.worker.mjs +46 -0
- package/dist/index.cjs +6229 -1096
- package/dist/index.d.cts +827 -3
- package/dist/index.d.ts +827 -3
- package/dist/index.js +6173 -1080
- package/docs/assistant.md +11 -7
- package/docs/bugs/2026-06-23-connectors-tab-out-of-scope.md +67 -0
- package/docs/bugs/2026-06-24-cloud-pooler-exhaustion.md +68 -0
- package/docs/bugs/2026-06-24-html-author-model-entitlement.md +77 -0
- package/docs/bugs/2026-06-25-chat-attached-files-not-connected.md +64 -0
- package/docs/connectors.md +166 -0
- package/package.json +7 -4
- package/docs/examples/dashboard.html +0 -284
package/docs/assistant.md
CHANGED
|
@@ -233,13 +233,17 @@ workspace's Secrets object.
|
|
|
233
233
|
|
|
234
234
|
## Voice (optional)
|
|
235
235
|
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
recording or transcribing, the composer is read-only** — it
|
|
241
|
-
"Listening… / Transcribing…" placeholder and the Send button is disabled —
|
|
242
|
-
the transcript is inserted when you stop.
|
|
236
|
+
The composer's 🎙 mic dictates **on-device** — speech is transcribed in your
|
|
237
|
+
browser by Whisper (WASM), so it needs no API key or setup and audio never leaves
|
|
238
|
+
your machine. There is no voice-provider choice in the GUI; the mic is always
|
|
239
|
+
shown when a microphone is available, and shown disabled with a tooltip when none
|
|
240
|
+
is. **While a note is recording or transcribing, the composer is read-only** — it
|
|
241
|
+
shows a "Listening… / Transcribing…" placeholder and the Send button is disabled —
|
|
242
|
+
and the transcript is inserted when you stop.
|
|
243
|
+
|
|
244
|
+
Keyed cloud transcription (`OPENAI_API_KEY` / Whisper or `ELEVENLABS_API_KEY`)
|
|
245
|
+
remains available to **API** callers via `POST /api/assistant/transcribe` for
|
|
246
|
+
backward compatibility; the GUI itself always dictates on-device.
|
|
243
247
|
|
|
244
248
|
## Cloud
|
|
245
249
|
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# Connectors settings tab rendered nothing (`fetchJson is not defined`)
|
|
2
|
+
|
|
3
|
+
**Date:** 2026-06-23
|
|
4
|
+
**Area:** GUI client script composition (`src/gui/app/modules/`)
|
|
5
|
+
**Severity:** broken feature (the entire Connectors settings tab was non-functional)
|
|
6
|
+
|
|
7
|
+
## Symptom
|
|
8
|
+
|
|
9
|
+
Opening **Settings → Connectors** in the GUI highlighted the tab but left the
|
|
10
|
+
drawer body showing whatever tab was previously open (e.g. the Lattice/Workspaces
|
|
11
|
+
panel). The connectors panel never appeared. The browser console showed an
|
|
12
|
+
uncaught `ReferenceError: fetchJson is not defined`.
|
|
13
|
+
|
|
14
|
+
## Root cause
|
|
15
|
+
|
|
16
|
+
The GUI client script (`appJs`) is composed by concatenating per-subsystem module
|
|
17
|
+
strings in `src/gui/app/modules/index.ts` and inlining the result into a single
|
|
18
|
+
`<script>` tag. The original modules' bodies execute inside a wrapper scope (an
|
|
19
|
+
IIFE/`DOMContentLoaded` closure), so their top-level functions — `fetchJson`,
|
|
20
|
+
`escapeHtml`, the `render*` panels, `selectDrawerTab` — are **scoped to the
|
|
21
|
+
wrapper**, not attached to `window`.
|
|
22
|
+
|
|
23
|
+
`connectorsSettingsJs` was appended as the **last** element of the composition
|
|
24
|
+
array — _after_ the wrapper closes. That placed `renderConnectorsPanel` at true
|
|
25
|
+
global scope. It was still reachable from `selectDrawerTab` (a wrapper-scoped
|
|
26
|
+
function can call a global one), so clicking the tab invoked it — but its very
|
|
27
|
+
first statement, `fetchJson('/api/connectors')`, referenced a name that only
|
|
28
|
+
exists inside the wrapper. The `ReferenceError` was thrown synchronously, before
|
|
29
|
+
the `.then`/`.catch` chain was attached and before `host.innerHTML` was written,
|
|
30
|
+
so the drawer body was never updated and the tab silently showed stale content.
|
|
31
|
+
|
|
32
|
+
Confirmed at runtime: `typeof window.renderConnectorsPanel === 'function'` while
|
|
33
|
+
`typeof window.fetchJson === 'undefined'` — the panel function was the only one
|
|
34
|
+
outside the wrapper.
|
|
35
|
+
|
|
36
|
+
## Fix
|
|
37
|
+
|
|
38
|
+
Move `connectorsSettingsJs` to sit **inside** the wrapper, next to `tableViewJs`
|
|
39
|
+
(which defines `selectDrawerTab`, the dispatcher that calls it). Function
|
|
40
|
+
declarations hoist within the wrapper, so `renderConnectorsPanel` can now see
|
|
41
|
+
`fetchJson`/`escapeHtml`, and `selectDrawerTab` can still call it. No change to
|
|
42
|
+
the panel code itself — purely a composition-order fix.
|
|
43
|
+
|
|
44
|
+
## Why existing tests missed it
|
|
45
|
+
|
|
46
|
+
`tests/unit/connectors-panel.test.ts` `eval`s `connectorsSettingsJs` in isolation
|
|
47
|
+
with **stubbed** `fetchJson`/`escapeHtml`, so it exercises the panel's markup and
|
|
48
|
+
button wiring but not the real composed-scope integration. The
|
|
49
|
+
`app-js-composition` test only pins length + hash, not execution. Neither could
|
|
50
|
+
observe the cross-module scope boundary.
|
|
51
|
+
|
|
52
|
+
## Lessons
|
|
53
|
+
|
|
54
|
+
- A module appended to the composition array is only correct if it lands on the
|
|
55
|
+
right side of the wrapper boundary. New client modules that depend on shared
|
|
56
|
+
helpers must be composed among the wrapped modules, not appended after them.
|
|
57
|
+
- Isolated-`eval` unit tests with stubbed globals cannot catch cross-module scope
|
|
58
|
+
bugs. Browser-level coverage is required for "does this tab actually render."
|
|
59
|
+
|
|
60
|
+
## Regression tests
|
|
61
|
+
|
|
62
|
+
- `tests/e2e/connectors-settings.spec.ts` — opens the drawer on another tab,
|
|
63
|
+
clicks the Connectors tab, and asserts the connectors panel + Jira credential
|
|
64
|
+
form render with **no page error**. Verified to fail when the module is appended
|
|
65
|
+
last (the bug) and pass when it is composed inside the wrapper (the fix).
|
|
66
|
+
- `tests/unit/app-js-composition.test.ts` — length/hash re-pinned after the
|
|
67
|
+
composition-order change.
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# Cloud queries fail under load with `EMAXCONNSESSION` (session-pooler exhaustion)
|
|
2
|
+
|
|
3
|
+
**Date:** 2026-06-24
|
|
4
|
+
**Area:** Postgres adapter / cloud connections (`src/db/postgres.ts`)
|
|
5
|
+
**Severity:** queries fail under load on small-pool cloud projects
|
|
6
|
+
|
|
7
|
+
## Symptom
|
|
8
|
+
|
|
9
|
+
On a cloud (Supabase) workspace, a burst of activity — e.g. the assistant
|
|
10
|
+
gathering data for a dashboard while the realtime broker is also running — caused
|
|
11
|
+
queries to fail with:
|
|
12
|
+
|
|
13
|
+
```
|
|
14
|
+
(EMAXCONNSESSION) max clients reached in session mode — max clients are limited to pool_size: 15
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
The assistant, seeing failed tool queries, paraphrased this as a "rate limit."
|
|
18
|
+
|
|
19
|
+
## Root cause
|
|
20
|
+
|
|
21
|
+
The workspace connection string pointed at Supabase's **session-mode pooler**
|
|
22
|
+
(`*.pooler.supabase.com:5432`), and the query pool connected there. In session
|
|
23
|
+
mode, every pooled client holds one upstream connection for its entire lifetime.
|
|
24
|
+
The app's footprint against that pooler was:
|
|
25
|
+
|
|
26
|
+
- the query `pg.Pool` (default `max` 10), plus
|
|
27
|
+
- the realtime broker's dedicated `LISTEN/NOTIFY` `pg.Client` (can't be pooled),
|
|
28
|
+
plus
|
|
29
|
+
- transient connections during workspace open (peek / converge / bootstrap).
|
|
30
|
+
|
|
31
|
+
Under a burst this exceeded the pooler's `pool_size` of 15. The visibility probe
|
|
32
|
+
(`changeVisibleToActiveRole`, fired per change on the **entire** cloud via the
|
|
33
|
+
global NOTIFY fan-out) competed for the same pool, amplifying the pressure.
|
|
34
|
+
|
|
35
|
+
This is a capacity mismatch, not a transient — so backoff/retry is the wrong fix;
|
|
36
|
+
it would just re-queue against a still-full pooler.
|
|
37
|
+
|
|
38
|
+
## Fix
|
|
39
|
+
|
|
40
|
+
Route the **query pool** through Supabase's **transaction-mode pooler** (port 6543) instead of the session pooler (5432). Transaction mode returns the upstream
|
|
41
|
+
connection to the pool at COMMIT, multiplexing many clients over far fewer
|
|
42
|
+
upstream slots. The adapter already holds no cross-statement session state (it
|
|
43
|
+
re-executes SQL per call rather than caching server-side prepared statements;
|
|
44
|
+
advisory locks are `_xact_` (transaction-scoped); `set_config(..., is_local=true)`
|
|
45
|
+
is transaction-local; `SET search_path` appears only inside `SECURITY DEFINER`
|
|
46
|
+
function bodies), so it is transaction-pooler-safe.
|
|
47
|
+
|
|
48
|
+
The realtime broker keeps its **session-mode** connection — `LISTEN/NOTIFY`
|
|
49
|
+
requires session mode, and it is a separate `pg.Client` not affected by the pool's
|
|
50
|
+
URL.
|
|
51
|
+
|
|
52
|
+
`toTransactionPoolerUrl()` does the rewrite surgically: only a Supabase pooler
|
|
53
|
+
host on :5432 is bumped to :6543 (host:port only, never userinfo); direct,
|
|
54
|
+
non-Supabase, already-:6543, and unparseable URLs are left untouched.
|
|
55
|
+
`LATTICE_PG_SESSION_POOLER=1` forces the pool back onto session mode.
|
|
56
|
+
|
|
57
|
+
## Lessons
|
|
58
|
+
|
|
59
|
+
- `EMAXCONNSESSION` is the session pooler's client cap, not your `pg.Pool` `max`
|
|
60
|
+
— count the dedicated `LISTEN` client + transient open-time connections too.
|
|
61
|
+
- Use the session pooler only for what needs it (LISTEN/NOTIFY); route ordinary
|
|
62
|
+
short queries through the transaction pooler.
|
|
63
|
+
|
|
64
|
+
## Regression tests
|
|
65
|
+
|
|
66
|
+
- `tests/unit/postgres-transaction-pooler.test.ts` — covers the rewrite
|
|
67
|
+
(Supabase :5432 → :6543, userinfo/db/query preserved) and the no-ops
|
|
68
|
+
(already-:6543, direct Supabase, non-Supabase, `:54321`, escape hatch).
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# HTML/dashboard authoring fails for connected Claude subscriptions (hardcoded model not entitled)
|
|
2
|
+
|
|
3
|
+
**Date:** 2026-06-24
|
|
4
|
+
**Area:** GUI assistant — delegated HTML authoring (`src/gui/ai/html-author.ts`)
|
|
5
|
+
**Severity:** broken feature (every `create_html_file` / `edit_html_file` failed for subscription auth)
|
|
6
|
+
|
|
7
|
+
## Symptom
|
|
8
|
+
|
|
9
|
+
Asking the assistant to build a dashboard ("Build me a dashboard that shows
|
|
10
|
+
contract value by client") never produced a file. The chat would gather the data,
|
|
11
|
+
say "Now I'll create the dashboard," then retry and stall. It failed on **both
|
|
12
|
+
local (SQLite) and cloud (Postgres) workspaces**, which ruled out the database.
|
|
13
|
+
On the cloud it was mislabeled as a "rate limit"; on local it just looped.
|
|
14
|
+
|
|
15
|
+
## Root cause
|
|
16
|
+
|
|
17
|
+
The chat assistant runs on `DEFAULT_MODEL` (`claude-haiku-4-5`). The HTML author
|
|
18
|
+
**hardcoded a different model**, `claude-sonnet-4-6`, for the delegated authoring
|
|
19
|
+
sub-call — using the **same resolved Claude auth** as the chat.
|
|
20
|
+
|
|
21
|
+
When the user is connected via a Claude **subscription** ("Connect with Claude" /
|
|
22
|
+
OAuth), the auth is entitled only to the models on that plan. This subscription
|
|
23
|
+
had `claude-haiku-4-5` but **not** `claude-sonnet-4-6`. Anthropic surfaces a
|
|
24
|
+
non-entitled model as a `429 rate_limit_error` on **every** call — even a
|
|
25
|
+
one-token one — not as a 403/404. So every authoring sub-call 429'd instantly and
|
|
26
|
+
no HTML file was ever produced. Because it's auth/model-based, it failed
|
|
27
|
+
identically on local and cloud.
|
|
28
|
+
|
|
29
|
+
Verified live over the real subscription auth:
|
|
30
|
+
|
|
31
|
+
```
|
|
32
|
+
haiku-4-5 : OK
|
|
33
|
+
sonnet-4-6 : 429 rate_limit_error (even max_tokens=16)
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Backoff/retry could never fix this — the 429 is permanent for a non-entitled
|
|
37
|
+
model, not transient.
|
|
38
|
+
|
|
39
|
+
(Separately, the cloud data-gathering step hit Supabase session-pooler exhaustion
|
|
40
|
+
— `EMAXCONNSESSION`, pool_size 15 — which is a distinct issue tracked on its own.)
|
|
41
|
+
|
|
42
|
+
## Fix
|
|
43
|
+
|
|
44
|
+
The author model must be one the resolved auth can actually call, and ideally the
|
|
45
|
+
strongest such model. `htmlAuthorModelForAuth(auth)` picks by **auth kind**: a
|
|
46
|
+
stronger model (`claude-sonnet-4-6`) for an Anthropic **API key** (entitled to all
|
|
47
|
+
GA models — restoring the strong authoring the feature was designed around), and
|
|
48
|
+
the **chat model** (`DEFAULT_MODEL`, proven entitled in-session) for an OAuth
|
|
49
|
+
**subscription** (whose entitlements vary; a non-entitled model 429s every call).
|
|
50
|
+
The authoring is still a focused, delegated sub-call (its own system prompt + a
|
|
51
|
+
larger `maxTokens`) — it just never assumes a model the auth can't run.
|
|
52
|
+
|
|
53
|
+
**Caveat — Haiku-only subscriptions.** Some "Connect with Claude" subscriptions
|
|
54
|
+
entitle _only_ `claude-haiku-4-5` (verified: every Opus/Sonnet model 429s). On
|
|
55
|
+
those, authoring uses Haiku and works, but Haiku is the weakest model and is not
|
|
56
|
+
reliable for multi-step agentic editing (it returns valid HTML but may not honor
|
|
57
|
+
the change, and may narrate an action without calling the tool). For dependable
|
|
58
|
+
authoring/editing, use an Anthropic **API key** (Settings → Assistant → Advanced),
|
|
59
|
+
which entitles the stronger model.
|
|
60
|
+
|
|
61
|
+
## Lessons
|
|
62
|
+
|
|
63
|
+
- Never hardcode a model for a sub-call that differs from the model the active
|
|
64
|
+
auth has already demonstrated it can use. A connected subscription's entitlement
|
|
65
|
+
is narrower than an API key's, and a non-entitled model returns `429
|
|
66
|
+
rate_limit_error` (looks like a transient rate limit but is permanent).
|
|
67
|
+
- A 429 that reproduces on a one-token request is an entitlement/permission
|
|
68
|
+
signal, not a load signal — do not back off; pick a usable model.
|
|
69
|
+
|
|
70
|
+
## Regression tests
|
|
71
|
+
|
|
72
|
+
- `tests/unit/gui-ai-html-author.test.ts` — asserts the authoring sub-call uses
|
|
73
|
+
`DEFAULT_MODEL` (the chat model) and explicitly **not** a hardcoded
|
|
74
|
+
`claude-sonnet-4-6`. (The prior test asserted the sonnet model — it encoded the
|
|
75
|
+
bug; it now guards against re-introducing a divergent, possibly-unentitled
|
|
76
|
+
model.) Verified live end-to-end: with the fix, `generateHtmlFile` returns a
|
|
77
|
+
valid HTML document over the same subscription that 429'd before.
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# Chat message not connected to its attached files
|
|
2
|
+
|
|
3
|
+
**Date:** 2026-06-25
|
|
4
|
+
**Area:** GUI assistant composer (`src/gui/app/modules/create-database-wizard.ts`, `onboarding.ts`) + chat route (`src/gui/chat-routes.ts`)
|
|
5
|
+
|
|
6
|
+
## Symptom
|
|
7
|
+
|
|
8
|
+
A user attached three screenshots to a chat message ("analyze these screenshots")
|
|
9
|
+
and the assistant replied that it did **not see any attached files** — even though
|
|
10
|
+
the files were referenced in its context. The attachment and the message were
|
|
11
|
+
disconnected: the assistant had no signal that those specific files belonged to
|
|
12
|
+
this request.
|
|
13
|
+
|
|
14
|
+
## Root cause
|
|
15
|
+
|
|
16
|
+
The composer Send fired two independent, unordered actions:
|
|
17
|
+
|
|
18
|
+
1. `sendStaged()` → `uploadFiles()` — a fire-and-forget ingest of the staged files.
|
|
19
|
+
2. `sendChat(text)` — POST `/api/chat` with the message only.
|
|
20
|
+
|
|
21
|
+
So the chat turn was sent **before** the files finished ingesting, and it carried
|
|
22
|
+
**no reference** to them. Nothing told the assistant "these just-attached files are
|
|
23
|
+
what the request is about." The earlier instinct — "send the images to the model as
|
|
24
|
+
vision" — was the wrong fix: it only helps images, breaks on multiple/complex file
|
|
25
|
+
types, and treats the symptom (analysis) rather than the cause (the message is not
|
|
26
|
+
linked to the files the user added).
|
|
27
|
+
|
|
28
|
+
## Fix
|
|
29
|
+
|
|
30
|
+
Make the one composer Send a connected, ordered workflow that works for **any** file
|
|
31
|
+
type and count, reusing the assistant's existing file-interaction tools:
|
|
32
|
+
|
|
33
|
+
1. **Add the files first.** `uploadFiles()` now returns a promise resolving with
|
|
34
|
+
`[{id, name}]` for each ingested file. `submitComposer` awaits the ingest before
|
|
35
|
+
sending the chat (`opts.silent` suppresses the single-file open-the-record jump
|
|
36
|
+
when a message accompanies the upload).
|
|
37
|
+
2. **Reference them in the turn.** `sendChat(text, attachedFiles)` includes the
|
|
38
|
+
ingested ids in the `/api/chat` body.
|
|
39
|
+
3. **Connect server-side.** `buildAttachedFilesNote(db, attachedFiles)` grounds each
|
|
40
|
+
id against the **visible** files table (a stale/invented id is dropped, never
|
|
41
|
+
referenced) and prefixes a note — "the user just attached these files … read them
|
|
42
|
+
with your file tools and use them to do what the user asks" — to the model's turn
|
|
43
|
+
only. The persisted user message is unchanged; the dispatch/tools still see the
|
|
44
|
+
real message (so `ingest_url` is unaffected).
|
|
45
|
+
|
|
46
|
+
The assistant then uses its existing tools (`get_row_context`, the `files` table,
|
|
47
|
+
each file's extracted text/description) to act on exactly the attached files.
|
|
48
|
+
|
|
49
|
+
## Lessons learned
|
|
50
|
+
|
|
51
|
+
- A "send X to the model" reflex narrows a general problem to one data type. The
|
|
52
|
+
generic, scalable fix connected the message to the files via ids and let the
|
|
53
|
+
existing per-file capabilities do the rest.
|
|
54
|
+
- Fire-and-forget + a dependent action is an ordering bug. Await the prerequisite
|
|
55
|
+
(ingest) before the dependent step (the chat turn) when one needs the other.
|
|
56
|
+
- Ground model-facing references against the visible DB so the assistant never
|
|
57
|
+
receives an id it can't actually see.
|
|
58
|
+
|
|
59
|
+
## Regression tests
|
|
60
|
+
|
|
61
|
+
- `tests/unit/chat-attached-files.test.ts` — `buildAttachedFilesNote`: empty when
|
|
62
|
+
nothing is attached; names a single file (singular phrasing) with its id; names
|
|
63
|
+
multiple files (plural phrasing, the reported multi-screenshot case); drops
|
|
64
|
+
stale/invisible ids instead of inventing a reference.
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
# Connectors
|
|
2
|
+
|
|
3
|
+
Connectors sync data from external systems into Lattice as **connected data
|
|
4
|
+
types** — tables whose rows are ingested from a source rather than authored
|
|
5
|
+
locally. They make external data first-class Lattice data: queryable, full-text
|
|
6
|
+
searchable, rendered to context, linked on the graph, and ACL-scoped on a cloud.
|
|
7
|
+
|
|
8
|
+
A _connector_ talks to one external product (a _toolkit_) and exposes its object
|
|
9
|
+
types as connected data types. The built-in connector is **Jira**, which talks to
|
|
10
|
+
Jira Cloud's REST + Agile APIs directly via [`jira.js`](https://github.com/MrRefactoring/jira.js)
|
|
11
|
+
using your own Atlassian credentials — there is no broker service and no extra
|
|
12
|
+
API key. Adding another source (Gmail, Slack, …) is a new `Connector`
|
|
13
|
+
implementation, not changes to the core.
|
|
14
|
+
|
|
15
|
+
> `jira.js` is an **optional dependency**. The package compiles and runs without
|
|
16
|
+
> it; it is loaded lazily and a clear error is thrown only when the Jira connector
|
|
17
|
+
> is actually used. Install it to use the connector: `npm install jira.js`.
|
|
18
|
+
|
|
19
|
+
## Concepts
|
|
20
|
+
|
|
21
|
+
| Concept | Meaning |
|
|
22
|
+
| ----------------------- | ---------------------------------------------------------------------------------------------------------------------- |
|
|
23
|
+
| **Connector** | An implementation of the fetch/auth SPI (e.g. the Jira connector). |
|
|
24
|
+
| **Toolkit** | An external product a connector serves (e.g. `jira`). |
|
|
25
|
+
| **Connected data type** | A Lattice table with a `source` descriptor — its rows are synced from a toolkit model. |
|
|
26
|
+
| **Connector instance** | A registered connection (`__lattice_connectors` row): which toolkit, the per-member connection handle, and sync state. |
|
|
27
|
+
|
|
28
|
+
A connected table's **natural key is its primary key** (a stable external id), so
|
|
29
|
+
re-syncs upsert idempotently. Every row carries connector lineage:
|
|
30
|
+
`_source_connector_id` and `_source_model` (immutable) plus `_source_synced_at`.
|
|
31
|
+
|
|
32
|
+
## Credentials
|
|
33
|
+
|
|
34
|
+
The Jira connector authenticates as **you**, with an Atlassian **API token**
|
|
35
|
+
(HTTP Basic auth: your account email + the token). Create one at
|
|
36
|
+
<https://id.atlassian.com/manage-profile/security/api-tokens>. Lattice validates
|
|
37
|
+
the credentials against Jira on connect (`GET /myself`) and stores them in the
|
|
38
|
+
**machine-local encrypted credential store** — they never leave the machine
|
|
39
|
+
except to call the Jira API directly, and they are never written to the registry,
|
|
40
|
+
logs, or any rendered context.
|
|
41
|
+
|
|
42
|
+
## Using the GUI
|
|
43
|
+
|
|
44
|
+
Open **Settings → Connectors**, fill in your Jira **site URL**
|
|
45
|
+
(`https://your-domain.atlassian.net`), **account email**, and **API token**, then
|
|
46
|
+
**Connect**. The credentials are validated and the initial sync runs. Each
|
|
47
|
+
connected toolkit shows its status + last-synced time with **Refresh** and
|
|
48
|
+
**Disconnect** buttons, and connected data types get a "Connected" badge in the
|
|
49
|
+
Objects list.
|
|
50
|
+
|
|
51
|
+
## Quick start (programmatic)
|
|
52
|
+
|
|
53
|
+
```typescript
|
|
54
|
+
import {
|
|
55
|
+
JiraConnector,
|
|
56
|
+
createConnector,
|
|
57
|
+
syncConnector,
|
|
58
|
+
syncIfStale,
|
|
59
|
+
disconnectConnector,
|
|
60
|
+
} from 'latticesql';
|
|
61
|
+
|
|
62
|
+
const connector = new JiraConnector();
|
|
63
|
+
|
|
64
|
+
// Validate + store the member's Atlassian credentials, returning a connection handle:
|
|
65
|
+
const { connectionId, displayName } = await connector.connect({
|
|
66
|
+
site: 'https://your-domain.atlassian.net',
|
|
67
|
+
email: 'you@example.com',
|
|
68
|
+
apiToken: process.env.JIRA_API_TOKEN!,
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
const connectorId = await createConnector(db, {
|
|
72
|
+
connector: 'jira',
|
|
73
|
+
toolkit: 'jira',
|
|
74
|
+
displayName: displayName ?? 'jira',
|
|
75
|
+
connectionRef: connectionId,
|
|
76
|
+
connectedBy: 'user-123',
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
await syncConnector(db, connector, connectorId); // defines the tables + ingests
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## Sync model
|
|
83
|
+
|
|
84
|
+
- **`syncConnector(db, connector, connectorId)`** — full sync: paginated +
|
|
85
|
+
bounded fetch per model, idempotent upsert with lineage stamping, soft-delete
|
|
86
|
+
of rows that vanished from the source, and graph-edge derivation from FK
|
|
87
|
+
columns. Records the outcome on the connector; an external-sync failure is
|
|
88
|
+
re-thrown (never swallowed).
|
|
89
|
+
- **`syncIfStale(db, connector, connectorId, maxAgeMs?)`** — no-op if the last
|
|
90
|
+
sync is within `maxAgeMs` (default 1 hour). Call it on app load.
|
|
91
|
+
- **`syncStaleConnectors(db, connector, maxAgeMs?)`** — sync every stale
|
|
92
|
+
connector this implementation serves. The GUI calls this on load, so connected
|
|
93
|
+
data refreshes hourly **without a scheduler**. Manual refresh forces a sync.
|
|
94
|
+
|
|
95
|
+
Reads done by the sync engine are bounded and column-projected — it never scans a
|
|
96
|
+
full table to diff. Per-parent models iterate the parent's already-synced keys
|
|
97
|
+
(comments are fetched per issue; sprints per board).
|
|
98
|
+
|
|
99
|
+
## Disconnecting
|
|
100
|
+
|
|
101
|
+
```typescript
|
|
102
|
+
await disconnectConnector(db, connector, connectorId, { outputDir });
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Soft-deletes every ingested row (children before parents), prunes rendered
|
|
106
|
+
context files, marks the connector `disconnected` (use `{ mode: 'hard' }` to also
|
|
107
|
+
remove the registry row), and drops the stored credentials. Soft-deleted rows
|
|
108
|
+
drop out of the rendered context, full-text search, and the GUI listings (all of
|
|
109
|
+
which filter `deleted_at IS NULL`), and their graph edges are removed — so the
|
|
110
|
+
data is no longer available to the agent while remaining physically present and
|
|
111
|
+
restorable.
|
|
112
|
+
|
|
113
|
+
## Cloud ACL
|
|
114
|
+
|
|
115
|
+
On a cloud (Postgres) workspace, the **owner** scopes connected data per member:
|
|
116
|
+
|
|
117
|
+
```typescript
|
|
118
|
+
await enableConnectorRls(db, connector, 'jira');
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
This enables Row-Level Security on the registry and the toolkit's connected
|
|
122
|
+
tables and applies each type's default visibility — `private` (visible only to
|
|
123
|
+
the connecting member) or `everyone` (shared with the team). It is a no-op on
|
|
124
|
+
SQLite, a non-cloud Postgres, or for a non-owner role.
|
|
125
|
+
|
|
126
|
+
Because each member connects with their own Atlassian credentials, the data a
|
|
127
|
+
member syncs is already scoped to what they can see in the source — per-user
|
|
128
|
+
auth, not a shared admin credential. Derived enrichment over connected rows
|
|
129
|
+
inherits the source's visibility automatically: write it through
|
|
130
|
+
`db.observe(table, pk, { changeKind: 'derived', sourceRef: [connectedRowId] })`,
|
|
131
|
+
and the source-gated fold hides it from a viewer who can't see the source.
|
|
132
|
+
|
|
133
|
+
## The Jira toolkit
|
|
134
|
+
|
|
135
|
+
Connecting Jira creates six connected data types:
|
|
136
|
+
|
|
137
|
+
| Table | Natural key | Notes |
|
|
138
|
+
| --------------- | ----------- | ------------------------------------------------------------- |
|
|
139
|
+
| `jira_projects` | project key | FTS on name/description |
|
|
140
|
+
| `jira_issues` | issue key | FTS on summary/description; edges → project, assignee, sprint |
|
|
141
|
+
| `jira_comments` | comment id | fetched per issue; edges → issue, author |
|
|
142
|
+
| `jira_users` | account id | |
|
|
143
|
+
| `jira_boards` | board id | edge → project |
|
|
144
|
+
| `jira_sprints` | sprint id | fetched per board; edge → board |
|
|
145
|
+
|
|
146
|
+
## Adding a connector
|
|
147
|
+
|
|
148
|
+
Implement the `Connector` SPI and point the GUI/registry at it. The SPI is small:
|
|
149
|
+
|
|
150
|
+
- `connector` / `toolkits()` / `models(toolkit)` — identity + the connected
|
|
151
|
+
`ConnectedModelDef`s (table schema, natural key, FK relations → graph edges).
|
|
152
|
+
- `listChanges(toolkit, model, ctx)` — an async iterable of normalized
|
|
153
|
+
`ExternalRecord`s for one model, **paged and bounded** (per-parent models read
|
|
154
|
+
`ctx.parentKey`).
|
|
155
|
+
- `disconnect(connectionRef)` — revoke / drop the stored connection.
|
|
156
|
+
- `authorize` / `completeAuth` — for an OAuth-redirect source. A
|
|
157
|
+
credential-based connector (like Jira) instead exposes a `connect(creds)`
|
|
158
|
+
method that validates + stores the credentials and returns a connection handle;
|
|
159
|
+
the GUI route calls it when the connector supports it.
|
|
160
|
+
|
|
161
|
+
The sync engine, graph wiring, teardown, and ACL all work from the model
|
|
162
|
+
descriptors — no further code is required.
|
|
163
|
+
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
```
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "latticesql",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.3.1",
|
|
4
4
|
"description": "Persistent structured memory for AI agent systems — pluggable SQLite or Postgres backend, LLM context bridge",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
"node": ">=18"
|
|
30
30
|
},
|
|
31
31
|
"scripts": {
|
|
32
|
-
"build": "tsup",
|
|
32
|
+
"build": "tsup && node scripts/build-gui-assets.mjs",
|
|
33
33
|
"typecheck": "tsc --noEmit",
|
|
34
34
|
"lint": "eslint src tests scripts",
|
|
35
35
|
"lint:fix": "eslint src tests scripts --fix",
|
|
@@ -47,8 +47,9 @@
|
|
|
47
47
|
"desktop:version:check": "node scripts/sync-desktop-version.mjs --check",
|
|
48
48
|
"test:deno": "deno test --no-check -A tests/deno/",
|
|
49
49
|
"desktop:dev": "npm run build && node scripts/sync-desktop-version.mjs && deno desktop --no-check --allow-all desktop/main.ts",
|
|
50
|
-
"desktop:build:mac": "npm run build && node scripts/sync-desktop-version.mjs && deno desktop --no-check --allow-env --allow-read --allow-write --allow-net --allow-run --allow-sys --allow-ffi --output dist-desktop/Lattice.dmg desktop/main.ts",
|
|
51
|
-
"desktop:build:
|
|
50
|
+
"desktop:build:mac": "npm run build && node scripts/sync-desktop-version.mjs && deno desktop --no-check --allow-env --allow-read --allow-write --allow-net --allow-run --allow-sys --allow-ffi --include dist/gui-assets --output dist-desktop/Lattice.dmg desktop/main.ts",
|
|
51
|
+
"desktop:build:mac:pkg": "npm run build && node scripts/sync-desktop-version.mjs && deno desktop --no-check --allow-env --allow-read --allow-write --allow-net --allow-run --allow-sys --allow-ffi --include dist/gui-assets --output dist-desktop/Lattice.app desktop/main.ts && sh scripts/build-mac-pkg.sh",
|
|
52
|
+
"desktop:build:win": "npm run build && node scripts/sync-desktop-version.mjs && deno desktop --no-check --allow-env --allow-read --allow-write --allow-net --allow-run --allow-sys --allow-ffi --include dist/gui-assets --output dist-desktop/Lattice.msi desktop/main.ts",
|
|
52
53
|
"prepublishOnly": "npm run check:generic && npm run build && npm run typecheck && npm test"
|
|
53
54
|
},
|
|
54
55
|
"dependencies": {
|
|
@@ -74,6 +75,7 @@
|
|
|
74
75
|
"optionalDependencies": {
|
|
75
76
|
"@aws-sdk/client-s3": "^3.1067.0",
|
|
76
77
|
"exceljs": "^4.4.0",
|
|
78
|
+
"jira.js": "^5.3.1",
|
|
77
79
|
"pg": "^8.11.0",
|
|
78
80
|
"pgvector": "^0.2.0",
|
|
79
81
|
"playwright": "^1.48.0",
|
|
@@ -82,6 +84,7 @@
|
|
|
82
84
|
},
|
|
83
85
|
"devDependencies": {
|
|
84
86
|
"@eslint/js": "^9.0.0",
|
|
87
|
+
"@huggingface/transformers": "^4.2.0",
|
|
85
88
|
"@playwright/test": "^1.48.0",
|
|
86
89
|
"@types/better-sqlite3": "^7.6.12",
|
|
87
90
|
"@types/jsdom": "^28.0.3",
|