okengine 0.18.4 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +18 -10
- package/README.md +2 -2
- package/package.json +51 -50
- package/site/content/docs/ai/index.mdx +25 -6
- package/site/content/docs/ai/llms-txt.mdx +1 -1
- package/site/content/docs/ai/meta.json +1 -1
- package/site/content/docs/ai/skills.mdx +4 -1
- package/site/content/docs/ai/try-it.mdx +58 -0
- package/site/content/docs/client/auth.mdx +208 -0
- package/site/content/docs/client/calling.mdx +451 -0
- package/site/content/docs/client/index.mdx +209 -0
- package/site/content/docs/client/live.mdx +234 -0
- package/site/content/docs/client/meta.json +5 -0
- package/site/content/docs/client/react.mdx +249 -0
- package/site/content/docs/elements/ai/agents.mdx +232 -0
- package/site/content/docs/elements/ai/index.mdx +341 -0
- package/site/content/docs/elements/ai/mcp.mdx +275 -0
- package/site/content/docs/elements/ai/meta.json +5 -0
- package/site/content/docs/elements/ai/models.mdx +283 -0
- package/site/content/docs/elements/ai/prompts.mdx +256 -0
- package/site/content/docs/elements/channel/email.mdx +266 -0
- package/site/content/docs/elements/channel/index.mdx +344 -0
- package/site/content/docs/elements/channel/meta.json +5 -0
- package/site/content/docs/elements/channel/push.mdx +218 -0
- package/site/content/docs/elements/channel/receipts.mdx +206 -0
- package/site/content/docs/elements/channel/sms.mdx +264 -0
- package/site/content/docs/elements/channel/whatsapp.mdx +220 -0
- package/site/content/docs/elements/clock/index.mdx +300 -0
- package/site/content/docs/elements/clock/meta.json +5 -0
- package/site/content/docs/elements/clock/schedules.mdx +508 -0
- package/site/content/docs/elements/clock/sleep.mdx +445 -0
- package/site/content/docs/elements/flow/consumers.mdx +707 -0
- package/site/content/docs/elements/flow/http.mdx +1080 -0
- package/site/content/docs/elements/flow/index.mdx +618 -0
- package/site/content/docs/elements/flow/meta.json +5 -0
- package/site/content/docs/elements/flow/routing.mdx +559 -0
- package/site/content/docs/elements/flow/workflows.mdx +640 -0
- package/site/content/docs/elements/gate/auth.mdx +371 -0
- package/site/content/docs/elements/gate/authorization.mdx +298 -0
- package/site/content/docs/elements/gate/index.mdx +418 -0
- package/site/content/docs/elements/gate/meta.json +5 -0
- package/site/content/docs/elements/gate/rate-limits.mdx +365 -0
- package/site/content/docs/elements/gate/rls.mdx +303 -0
- package/site/content/docs/elements/gate/tenancy.mdx +372 -0
- package/site/content/docs/elements/index.mdx +52 -11
- package/site/content/docs/elements/signal/broadcast.mdx +517 -0
- package/site/content/docs/elements/signal/index.mdx +352 -0
- package/site/content/docs/elements/signal/live.mdx +590 -0
- package/site/content/docs/elements/signal/meta.json +5 -0
- package/site/content/docs/elements/signal/once.mdx +596 -0
- package/site/content/docs/elements/store/files.mdx +659 -0
- package/site/content/docs/elements/store/index.mdx +310 -0
- package/site/content/docs/elements/store/kv.mdx +629 -0
- package/site/content/docs/elements/store/meta.json +5 -0
- package/site/content/docs/elements/store/search.mdx +946 -0
- package/site/content/docs/elements/store/sql.mdx +937 -0
- package/site/content/docs/elements/vault/config.mdx +256 -0
- package/site/content/docs/elements/vault/index.mdx +356 -0
- package/site/content/docs/elements/vault/meta.json +5 -0
- package/site/content/docs/elements/vault/rotation.mdx +273 -0
- package/site/content/docs/elements/vault/secrets.mdx +285 -0
- package/site/content/docs/index.mdx +78 -30
- package/site/content/docs/meta.json +11 -8
- package/site/content/docs/plugins/cors.mdx +2 -1
- package/site/content/docs/plugins/csrf.mdx +4 -1
- package/site/content/docs/plugins/headers.mdx +2 -1
- package/site/content/docs/plugins/ip-allowlist.mdx +2 -1
- package/site/content/docs/plugins/magic-link.mdx +6 -0
- package/site/content/docs/plugins/maintenance-mode.mdx +2 -1
- package/site/content/docs/plugins/oauth.mdx +1 -1
- package/site/content/docs/plugins/otp.mdx +12 -1
- package/site/content/docs/plugins/passkey.mdx +39 -15
- package/site/content/docs/plugins/two-factor.mdx +91 -34
- package/site/content/docs/plugins/username.mdx +1 -1
- package/site/content/docs/recipes/caddy.mdx +3 -3
- package/site/content/docs/recipes/index.mdx +18 -28
- package/site/content/docs/recipes/mailpit.mdx +1 -1
- package/site/content/docs/recipes/meilisearch.mdx +3 -3
- package/site/content/docs/recipes/meta.json +1 -4
- package/site/content/docs/recipes/nginx.mdx +2 -2
- package/site/content/docs/recipes/openrouter.mdx +283 -0
- package/site/content/docs/recipes/pgdog.mdx +3 -3
- package/site/content/docs/recipes/postgres.mdx +1 -2
- package/site/content/docs/recipes/rustfs.mdx +2 -2
- package/site/content/docs/recipes/traefik.mdx +3 -3
- package/site/content/docs/reference/cli.mdx +231 -0
- package/site/content/docs/reference/configuration.mdx +30 -10
- package/site/content/docs/reference/environment-variables.mdx +93 -20
- package/site/content/docs/reference/errors.mdx +132 -29
- package/site/content/docs/reference/fx.mdx +137 -73
- package/site/content/docs/reference/index.mdx +13 -21
- package/site/content/docs/reference/meta.json +4 -5
- package/site/content/docs/reference/okid.mdx +42 -17
- package/site/content/docs/reference/plugins.mdx +4 -3
- package/site/content/docs/reference/security.mdx +197 -0
- package/site/content/docs/understand/meta.json +5 -0
- package/site/content/docs/understand/the-anatomy.mdx +132 -0
- package/site/content/docs/understand/the-model.mdx +32 -0
- package/site/content/docs/understand/the-problem.mdx +74 -0
- package/site/content/docs/understand/the-vocabulary.mdx +26 -0
- package/src/auth/api-keys.ts +2 -1
- package/src/auth/bindings.ts +64 -16
- package/src/auth/gate-auth.test.ts +6 -1
- package/src/auth/identity.ts +152 -5
- package/src/auth/index.ts +29 -0
- package/src/auth/invites.ts +2 -1
- package/src/auth/method-context.ts +41 -0
- package/src/auth/oauth-as/crypto.ts +2 -1
- package/src/auth/oauth-as/stores.ts +2 -1
- package/src/auth/operator.ts +2 -1
- package/src/auth/sessions-jwt.test.ts +77 -0
- package/src/auth/sessions.ts +20 -4
- package/src/auth/tenants.ts +3 -2
- package/src/auth/two-factor-challenge.test.ts +61 -0
- package/src/auth/two-factor-challenge.ts +284 -0
- package/src/auth/verification.ts +5 -0
- package/src/bench/REPORT.md +49 -0
- package/src/bench/g03-signal-once.bench.ts +1 -1
- package/src/bench/g10-observability-contention.bench.ts +1 -1
- package/src/bench/g17-hybrid-search.bench.ts +572 -0
- package/src/bench/load-app.ts +2 -10
- package/src/cli/ai-setup/ai-setup.test.ts +331 -42
- package/src/cli/ai-setup/apply.ts +294 -52
- package/src/cli/ai-setup/catalog.ts +238 -1342
- package/src/cli/ai-setup/index.ts +14 -99
- package/src/cli/ai-setup/prompts.ts +60 -443
- package/src/cli/ask-seed.test.ts +96 -0
- package/src/cli/ask-seed.ts +82 -0
- package/src/cli/ask-vault-gaps.test.ts +84 -0
- package/src/cli/ask-vault-gaps.ts +146 -0
- package/src/cli/build.ts +2 -2
- package/src/cli/client-add.test.ts +26 -1
- package/src/cli/client-add.ts +106 -27
- package/src/cli/competitor-mention-removal.test.ts +1 -1
- package/src/cli/db-seed.ts +2 -0
- package/src/cli/db.ts +150 -11
- package/src/cli/dev.ts +119 -211
- package/src/cli/docker-clean.ts +2 -2
- package/src/cli/docker-cli.test.ts +1 -1
- package/src/cli/doctor-diff.ts +4 -2
- package/src/cli/doctor-pii.test.ts +1 -1
- package/src/cli/doctor.ts +14 -1
- package/src/cli/hero-meta.test.ts +4 -2
- package/src/cli/load-config.images.test.ts +8 -10
- package/src/cli/load-config.ts +1 -1
- package/src/cli/registry.ts +22 -10
- package/src/cli/replay.ts +3 -1
- package/src/cli/start.ts +1 -1
- package/src/cli/tui/keys.ts +2 -3
- package/src/client/auth/cookies.ts +74 -0
- package/src/client/auth/create-auth-client.ts +636 -0
- package/src/client/auth/denials.ts +99 -0
- package/src/client/auth/session.ts +283 -0
- package/src/client/auth.test.ts +251 -0
- package/src/client/auth.ts +39 -114
- package/src/client/create-with-session.ts +245 -0
- package/src/client/create.ts +51 -5
- package/src/client/index.ts +11 -1
- package/src/client/live.ts +11 -79
- package/src/client/notes-contract.test.ts +5 -10
- package/src/client/sse.ts +137 -0
- package/src/client/stream.ts +147 -0
- package/src/client/transport.test.ts +32 -0
- package/src/client/transport.ts +123 -26
- package/src/client/types.ts +118 -14
- package/src/client-react/index.ts +185 -24
- package/src/client-react/use-live-query.ts +13 -3
- package/src/compiler/aot.test.ts +7 -4
- package/src/compiler/effects-embed.test.ts +56 -0
- package/src/compiler/effects-fetch.test.ts +43 -0
- package/src/compiler/effects-infer.ts +58 -2
- package/src/compiler/extract.test.ts +258 -44
- package/src/compiler/extract.ts +566 -48
- package/src/compiler/fixtures/skyport/src/flows/bookings/index.ts +7 -4
- package/src/compiler/fixtures/skyport/src/flows/bookings/signals.ts +2 -8
- package/src/compiler/fixtures/skyport.expected.json +4 -4
- package/src/compiler/fixtures/triggers/five-triggers.ts +14 -10
- package/src/compiler/response.ts +2 -2
- package/src/compiler/search-writer-isolation.test.ts +40 -0
- package/src/config/index.ts +11 -0
- package/src/console/server/app.ts +17 -23
- package/src/console/server/bind.ts +4 -0
- package/src/console/server/console.test.ts +2 -2
- package/src/console/server/flows-invoke.test.ts +50 -34
- package/src/console/server/flows.ts +436 -255
- package/src/console/server/runs-ingest.test.ts +6 -3
- package/src/console/server/serve.ts +1 -1
- package/src/console/server/signals.test.ts +1 -5
- package/src/console/server/signals.ts +8 -6
- package/src/console/server/state.ts +6 -1
- package/src/console/server/store.ts +1 -1
- package/src/console/ui-next/dist/assets/FileExportIcon-Ck-5od4R.js +1 -0
- package/src/console/ui-next/dist/assets/MoreHorizontalCircle01Icon-gMNGsE37.js +1 -0
- package/src/console/ui-next/dist/assets/PlusSignIcon-CwG3nxfu.js +1 -0
- package/src/console/ui-next/dist/assets/UnavailableIcon-D9cvHVPr.js +1 -0
- package/src/console/ui-next/dist/assets/UserIcon-DaE2PB5_.js +1 -0
- package/src/console/ui-next/dist/assets/access-page-DFeymU07.js +4 -0
- package/src/console/ui-next/dist/assets/agent-disclosure-U1rdfblp.js +1 -0
- package/src/console/ui-next/dist/assets/cache-glyph-BeFJeqBG.js +1 -0
- package/src/console/ui-next/dist/assets/call-pii-button-CVAONPii.js +1 -0
- package/src/console/ui-next/dist/assets/collapsible-D2A6NJ-3.js +1 -0
- package/src/console/ui-next/dist/assets/copy-inline-button-CAYD18cr.js +1 -0
- package/src/console/ui-next/dist/assets/dagre.esm-B1_XeuLP.js +1 -0
- package/src/console/ui-next/dist/assets/detail-header-DVWNjWjg.js +1 -0
- package/src/console/ui-next/dist/assets/dropdown-menu-4h2LOVXM.js +1 -0
- package/src/console/ui-next/dist/assets/duration-tone-Cgk_h5ja.js +9 -0
- package/src/console/ui-next/dist/assets/element-icons-BI8cJgdh.js +1 -0
- package/src/console/ui-next/dist/assets/explorer-empty-CJs5A-wm.js +1 -0
- package/src/console/ui-next/dist/assets/flows-page-Dss7941e.js +1 -0
- package/src/console/ui-next/dist/assets/highlighted-json-MYZQtRnw.js +154 -0
- package/src/console/ui-next/dist/assets/http-method-Jrh39p7A.js +1 -0
- package/src/console/ui-next/dist/assets/index-D0zS5rKO.css +2 -0
- package/src/console/ui-next/dist/assets/index-DXP2dBIF.js +63 -0
- package/src/console/ui-next/dist/assets/observability-page-HvolXxTI.js +4 -0
- package/src/console/ui-next/dist/assets/react-dom-Ddte4I-Q.js +9 -0
- package/src/console/ui-next/dist/assets/replica-lag-CSh2dzrb.js +18 -0
- package/src/console/ui-next/dist/assets/request-meta-BatF8KrK.js +1 -0
- package/src/console/ui-next/dist/assets/shortcut-keys-3ILd8oGn.js +1 -0
- package/src/console/ui-next/dist/assets/{sql-BskegiFM.js → sql-BsFa4tDR.js} +1 -1
- package/src/console/ui-next/dist/assets/store-page-eiKiHnNe.js +41 -0
- package/src/console/ui-next/dist/assets/trace-detail-sheet-CZkMeKS-.js +2 -0
- package/src/console/ui-next/dist/assets/tree-expand-toggle-CW8y5A2h.js +55 -0
- package/src/console/ui-next/dist/assets/units-page-B_RWJrEO.js +1 -0
- package/src/console/ui-next/dist/assets/{use-vault-list-uk4WVboC.js → use-vault-list-CT4-gajj.js} +1 -1
- package/src/console/ui-next/dist/assets/vault-page-CWrg-A68.js +2 -0
- package/src/console/ui-next/dist/assets/xyflow-CSyC6ryz.css +1 -0
- package/src/console/ui-next/dist/assets/xyflow-yApv7D4e.js +7 -0
- package/src/console/ui-next/dist/index.html +6 -13
- package/src/console/ui-next/seed-invoke-host.ts +21 -18
- package/src/console/ui-next/src/client.ts +7 -2
- package/src/console/ui-next/src/features/flows/graph/element-map.ts +2 -0
- package/src/console/ui-next/src/features/flows/traces/effect-kind.ts +12 -2
- package/src/console/ui-next/src/features/flows/traces/effect-summary.ts +4 -0
- package/src/console/ui-next/src/features/flows/traces/trace-detail-sheet.tsx +8 -0
- package/src/console/ui-next/src/features/flows/traces/waterfall-bars.ts +4 -0
- package/src/console/ui-next/src/features/flows/traces/waterfall-tooltip.ts +9 -3
- package/src/console/ui-next/src/features/store/detail/reveal-cell.tsx +7 -11
- package/src/console/ui-next/src/features/store/detail/store-row-detail-sheet.tsx +2 -2
- package/src/console/ui-next/src/features/store/lib/fields-from-table.ts +2 -0
- package/src/console/ui-next/src/features/store/lib/grid-model.test.ts +25 -1
- package/src/console/ui-next/src/features/store/lib/grid-model.ts +36 -1
- package/src/console/ui-next/src/features/store/query/query-results.tsx +3 -3
- package/src/console/ui-next/src/features/units/detail/effects-summary.tsx +2 -0
- package/src/console/ui-next/src/features/units/lib/call-read-safe.ts +3 -1
- package/src/console/ui-next/src/features/units/lib/fields-from-schema.ts +6 -0
- package/src/docker/ai-model-status.test.ts +1 -12
- package/src/docker/ai-model-status.ts +10 -38
- package/src/docker/compose-up.test.ts +100 -0
- package/src/docker/compose-up.ts +292 -0
- package/src/docker/compose.ts +4 -14
- package/src/docker/derive.ts +9 -34
- package/src/docker/docker.test.ts +16 -261
- package/src/docker/images-config.test.ts +13 -25
- package/src/docker/index.ts +1 -24
- package/src/docker/recipes/index.ts +0 -23
- package/src/docker/recipes/pgdog.ts +6 -4
- package/src/docker/stack-id.test.ts +2 -2
- package/src/docker/stack-id.ts +0 -1
- package/src/docker/types.ts +2 -2
- package/src/drivers/ai-anthropic.ts +10 -0
- package/src/drivers/ai-openai-compatible.ts +34 -7
- package/src/drivers/ai-providers.test.ts +0 -107
- package/src/drivers/ai-stream.test.ts +0 -34
- package/src/drivers/ai-types.ts +17 -8
- package/src/drivers/channel-fcm.ts +6 -1
- package/src/drivers/channel-msegat.ts +6 -1
- package/src/drivers/channel-resend.ts +5 -1
- package/src/drivers/channel-smtp.ts +5 -1
- package/src/drivers/channel-sndr.ts +10 -1
- package/src/drivers/channel-taqnyat-mail.ts +5 -1
- package/src/drivers/channel-taqnyat-whatsapp.ts +6 -1
- package/src/drivers/channel-taqnyat.ts +6 -1
- package/src/drivers/channel-types.ts +10 -0
- package/src/drivers/channel-unifonic.ts +6 -1
- package/src/drivers/channel-wa-cloud.ts +6 -1
- package/src/drivers/channel-webpush.ts +6 -1
- package/src/drivers/external.ts +33 -0
- package/src/drivers/index.ts +0 -10
- package/src/drivers/meilisearch.ts +6 -0
- package/src/drivers/oauth-types.ts +4 -0
- package/src/drivers/postgres.test.ts +74 -3
- package/src/drivers/postgres.ts +175 -6
- package/src/drivers/signal-types.ts +5 -5
- package/src/drivers/types.ts +7 -0
- package/src/drivers/vault-types.ts +5 -0
- package/src/elements/ai/declare.ts +17 -4
- package/src/elements/ai/eval.ts +1 -1
- package/src/elements/ai/mcp-http.ts +1 -1
- package/src/elements/ai/pii.ts +1 -1
- package/src/elements/ai/providers.test.ts +289 -0
- package/src/elements/ai/providers.ts +169 -0
- package/src/elements/ai/runtime.ts +68 -5
- package/src/elements/ai/schema.ts +2 -2
- package/src/elements/ai.test.ts +7 -7
- package/src/elements/ai.ts +17 -1
- package/src/elements/channel/runtime.ts +105 -12
- package/src/elements/clock/cron-fields.test.ts +144 -0
- package/src/elements/clock/cron-fields.ts +185 -0
- package/src/elements/clock/declare.ts +247 -3
- package/src/elements/clock.ts +21 -1
- package/src/elements/index.ts +15 -0
- package/src/elements/signal/chaos-child.ts +1 -2
- package/src/elements/signal/declare.ts +65 -37
- package/src/elements/signal/delivery-modes.test.ts +11 -31
- package/src/elements/signal/dry-run-replay.test.ts +1 -7
- package/src/elements/signal/dry-run-write-isolation.test.ts +1 -7
- package/src/elements/signal/key-ordering.test.ts +6 -26
- package/src/elements/signal/lease-reclaim.test.ts +2 -6
- package/src/elements/signal/optional-emit.test.ts +6 -9
- package/src/elements/signal/order-lifecycle.test.ts +4 -17
- package/src/elements/signal/orphan-messages.test.ts +4 -15
- package/src/elements/signal/reconcile.test.ts +2 -2
- package/src/elements/signal/runtime.ts +1 -1
- package/src/elements/signal/schema-emit.test.ts +4 -8
- package/src/elements/signal.test.ts +24 -49
- package/src/elements/signal.ts +9 -2
- package/src/elements/store/domain-ddl.test.ts +1 -1
- package/src/elements/store/live-http.test.ts +9 -4
- package/src/elements/store/prepare-row.test.ts +123 -0
- package/src/elements/store/resource.ts +43 -12
- package/src/elements/store/schema-decl.ts +117 -0
- package/src/elements/store/search-backfill.ts +205 -0
- package/src/elements/store/search-bind.ts +66 -0
- package/src/elements/store/search-bm25.ts +78 -0
- package/src/elements/store/search-ddl.ts +156 -0
- package/src/elements/store/search-embed-flow.ts +141 -0
- package/src/elements/store/search-errors.ts +34 -0
- package/src/elements/store/search-fusion.ts +112 -0
- package/src/elements/store/search-lsh.ts +179 -0
- package/src/elements/store/search-runtime.ts +300 -0
- package/src/elements/store/search.test.ts +144 -0
- package/src/elements/store/sql-session.test.ts +1 -0
- package/src/elements/store/sql-session.ts +83 -3
- package/src/elements/store/table.ts +23 -2
- package/src/elements/store/upsert-app.test.ts +1 -3
- package/src/elements/store.ts +50 -0
- package/src/full.ts +4 -2
- package/src/http.ts +4 -1
- package/src/i18n/catalogs/ar.ts +14 -6
- package/src/i18n/catalogs/en.ts +14 -6
- package/src/index.ts +6 -1
- package/src/kernel/adopt-barrel-fresh.test.ts +7 -7
- package/src/kernel/adopt-routes.ts +9 -1
- package/src/kernel/app-auth.ts +5 -0
- package/src/kernel/app.ts +102 -9
- package/src/kernel/auto-cache.test.ts +6 -12
- package/src/kernel/auto-registry.test.ts +3 -3
- package/src/kernel/boot-bind/ai.test.ts +22 -19
- package/src/kernel/boot-bind/ai.ts +5 -43
- package/src/kernel/boot-bind/clock.ts +23 -5
- package/src/kernel/boot-bind/honor-config.test.ts +4 -4
- package/src/kernel/boot.test.ts +11 -6
- package/src/kernel/boot.ts +66 -16
- package/src/kernel/boundary-contract.ts +93 -0
- package/src/kernel/budget.test.ts +1 -1
- package/src/kernel/call.ts +89 -0
- package/src/kernel/capability.ts +6 -0
- package/src/kernel/client-descriptor.ts +123 -0
- package/src/kernel/clock-timezone.test.ts +80 -0
- package/src/kernel/correlation.test.ts +1 -1
- package/src/kernel/dry-run.ts +11 -8
- package/src/kernel/effects-stamping.test.ts +42 -9
- package/src/kernel/effects.test.ts +4 -2
- package/src/kernel/effects.ts +51 -9
- package/src/kernel/errors-channel.ts +17 -0
- package/src/kernel/errors-live-resume.ts +3 -2
- package/src/kernel/errors-tenant.ts +7 -4
- package/src/kernel/errors.registry-helpers.ts +96 -0
- package/src/kernel/errors.registry.test.ts +139 -25
- package/src/kernel/errors.ts +102 -23
- package/src/kernel/external-effects.test.ts +196 -0
- package/src/kernel/flow.test.ts +13 -4
- package/src/kernel/flow.ts +69 -71
- package/src/kernel/fx-ask-telemetry.test.ts +2 -2
- package/src/kernel/fx-dead-letters.test.ts +3 -3
- package/src/kernel/fx-fetch.ts +55 -0
- package/src/kernel/fx-live-stream.ts +2 -2
- package/src/kernel/fx-live.test.ts +8 -8
- package/src/kernel/fx.test.ts +17 -11
- package/src/kernel/fx.ts +180 -57
- package/src/kernel/horizontal-child.ts +1 -1
- package/src/kernel/http-query.test.ts +3 -6
- package/src/kernel/index.ts +13 -3
- package/src/kernel/instance-id.ts +1 -1
- package/src/kernel/instances.test.ts +40 -0
- package/src/kernel/instances.ts +47 -9
- package/src/kernel/live-http.test.ts +6 -7
- package/src/kernel/live-http.ts +6 -2
- package/src/kernel/live-resume.test.ts +1 -1
- package/src/kernel/mcp-tool.test.ts +3 -3
- package/src/kernel/on.ts +98 -4
- package/src/kernel/pipeline.test.ts +12 -15
- package/src/kernel/plugin-elements.test.ts +1 -1
- package/src/kernel/stamp-http.test.ts +3 -3
- package/src/kernel/triggers.ts +210 -95
- package/src/kernel-entry.ts +0 -1
- package/src/manifest/diff.ts +11 -1
- package/src/manifest/fixtures/skyport.excerpt.json +1 -1
- package/src/manifest/types.ts +46 -2
- package/src/mcp/docs-index.ts +3 -3
- package/src/mcp/docs-mcp.test.ts +2 -2
- package/src/mcp/docs-tools.ts +1 -1
- package/src/okid.test.ts +56 -0
- package/src/okid.ts +55 -11
- package/src/plugins/anonymous.ts +8 -4
- package/src/plugins/auth/shared.ts +27 -3
- package/src/plugins/auth-delivery.mailpit.integration.test.ts +1 -1
- package/src/plugins/auth-methods.security.test.ts +561 -23
- package/src/plugins/config-source.ts +1 -1
- package/src/plugins/csrf.test.ts +65 -0
- package/src/plugins/index.ts +2 -0
- package/src/plugins/magic-link.ts +36 -16
- package/src/plugins/oauth/shared.ts +3 -1
- package/src/plugins/oauth.ts +22 -14
- package/src/plugins/otp.ts +37 -21
- package/src/plugins/passkey-webauthn.ts +9 -6
- package/src/plugins/passkey.ts +98 -26
- package/src/plugins/pre-account-hijack.test.ts +223 -0
- package/src/plugins/two-factor.ts +466 -37
- package/src/plugins/username.ts +19 -7
- package/src/release/build-lib.ts +1 -0
- package/src/release/limits.ts +2 -2
- package/src/release/measure.ts +1 -1
- package/src/runs/collect.test.ts +0 -2
- package/src/runtime/json-code-block.test.ts +250 -20
- package/src/runtime/json-code-block.ts +1261 -41
- package/src/term.test.ts +1 -1
- package/src/test/create-test-app.test.ts +12 -12
- package/src/test/live-signals.test.ts +6 -6
- package/src/test/provisions.integration.test.ts +10 -14
- package/src/test/reset-element-registries.ts +1 -1
- package/src/test/tenant-isolation.test.ts +12 -6
- package/site/content/docs/deployment/docker-swarm.mdx +0 -164
- package/site/content/docs/deployment/docker.mdx +0 -227
- package/site/content/docs/deployment/index.mdx +0 -83
- package/site/content/docs/deployment/kubernetes.mdx +0 -176
- package/site/content/docs/deployment/meta.json +0 -5
- package/site/content/docs/deployment/reverse-proxy.mdx +0 -234
- package/site/content/docs/elements/ai.mdx +0 -385
- package/site/content/docs/elements/channel.mdx +0 -346
- package/site/content/docs/elements/clock.mdx +0 -244
- package/site/content/docs/elements/flow.mdx +0 -420
- package/site/content/docs/elements/gate.mdx +0 -436
- package/site/content/docs/elements/signal.mdx +0 -380
- package/site/content/docs/elements/store.mdx +0 -1099
- package/site/content/docs/elements/vault.mdx +0 -405
- package/site/content/docs/get-started/basic-usage.mdx +0 -173
- package/site/content/docs/get-started/index.mdx +0 -43
- package/site/content/docs/get-started/installation.mdx +0 -220
- package/site/content/docs/get-started/introduction.mdx +0 -132
- package/site/content/docs/get-started/meta.json +0 -13
- package/site/content/docs/get-started/project-structure.mdx +0 -925
- package/site/content/docs/get-started/testing.mdx +0 -328
- package/site/content/docs/get-started/why.mdx +0 -162
- package/site/content/docs/recipes/llama-cpp.mdx +0 -151
- package/site/content/docs/recipes/ollama.mdx +0 -142
- package/site/content/docs/recipes/sglang.mdx +0 -105
- package/site/content/docs/recipes/vllm.mdx +0 -106
- package/site/content/docs/reference/cli.md +0 -232
- package/site/content/docs/reference/client.mdx +0 -469
- package/site/content/docs/reference/security.md +0 -74
- package/src/cli/ai-setup/detect-ollama.ts +0 -213
- package/src/cli/ai-setup/recommend.test.ts +0 -225
- package/src/cli/ai-setup/recommend.ts +0 -225
- package/src/cli/dev-controls.test.ts +0 -122
- package/src/cli/dev-controls.ts +0 -164
- package/src/cli/tui/DevLive.tsx +0 -126
- package/src/console/ui-next/dist/assets/access-page-De7Lc2JC.js +0 -4
- package/src/console/ui-next/dist/assets/agent-disclosure-BP0Y0Sux.js +0 -1
- package/src/console/ui-next/dist/assets/cache-glyph-B5X-NM0-.js +0 -1
- package/src/console/ui-next/dist/assets/call-pii-button-ChAUiLo9.js +0 -1
- package/src/console/ui-next/dist/assets/collapsible-DGnOM2ph.js +0 -1
- package/src/console/ui-next/dist/assets/copy-inline-button-DBHhgHKP.js +0 -1
- package/src/console/ui-next/dist/assets/dagre.esm-ZwcdTuZZ.js +0 -1
- package/src/console/ui-next/dist/assets/detail-header-DhHM1iaZ.js +0 -1
- package/src/console/ui-next/dist/assets/dropdown-menu-_NjTEo5_.js +0 -1
- package/src/console/ui-next/dist/assets/duration-tone-sC3lGABz.js +0 -9
- package/src/console/ui-next/dist/assets/element-icons-BVXtRyd3.js +0 -1
- package/src/console/ui-next/dist/assets/explorer-empty-2uIhBu0_.js +0 -1
- package/src/console/ui-next/dist/assets/flows-page-RGy7VEA_.js +0 -1
- package/src/console/ui-next/dist/assets/highlighted-json-Awq7gYdu.js +0 -154
- package/src/console/ui-next/dist/assets/http-method-_UHM2ODJ.js +0 -1
- package/src/console/ui-next/dist/assets/index-Ck88Jmv8.css +0 -2
- package/src/console/ui-next/dist/assets/index-_rgpdVzo.js +0 -66
- package/src/console/ui-next/dist/assets/link-BX6Vqztd.js +0 -1
- package/src/console/ui-next/dist/assets/observability-page-Ds6pcnh-.js +0 -4
- package/src/console/ui-next/dist/assets/preload-helper-oH4irX4C.js +0 -1
- package/src/console/ui-next/dist/assets/react-D8E3mtu1.js +0 -1
- package/src/console/ui-next/dist/assets/react-dom-Bph1y7z7.js +0 -9
- package/src/console/ui-next/dist/assets/replica-lag-DKRrbvdo.js +0 -18
- package/src/console/ui-next/dist/assets/request-meta-DV0ywz7t.js +0 -1
- package/src/console/ui-next/dist/assets/shortcut-JQIZlWfm.js +0 -1
- package/src/console/ui-next/dist/assets/shortcut-keys-DKxNTe_m.js +0 -1
- package/src/console/ui-next/dist/assets/skeleton-D-czQJT6.js +0 -1
- package/src/console/ui-next/dist/assets/store-page-02xOiqIK.js +0 -41
- package/src/console/ui-next/dist/assets/trace-detail-sheet-B09O8rA6.js +0 -2
- package/src/console/ui-next/dist/assets/tree-expand-toggle-BtyhmWb4.js +0 -55
- package/src/console/ui-next/dist/assets/units-page-4rHOePuE.js +0 -1
- package/src/console/ui-next/dist/assets/useMutation-B8EO02Ej.js +0 -1
- package/src/console/ui-next/dist/assets/useRender-BE2A9BWC.js +0 -1
- package/src/console/ui-next/dist/assets/vault-page-DISPgxLM.js +0 -2
- package/src/console/ui-next/dist/assets/xyflow-D7n4g6go.js +0 -7
- package/src/console/ui-next/dist/assets/xyflow-DZ0Ws1xk.css +0 -1
- package/src/docker/ollama-pull.ts +0 -232
- package/src/docker/recipes/llama-cpp.ts +0 -298
- package/src/docker/recipes/ollama.ts +0 -44
- package/src/docker/recipes/sglang.ts +0 -55
- package/src/docker/recipes/vllm.ts +0 -44
- package/src/drivers/ai-ollama-tools.integration.test.ts +0 -109
- package/src/drivers/ai-ollama.integration.test.ts +0 -184
- package/src/drivers/ai-ollama.ts +0 -389
- package/src/drivers/ollama.ts +0 -14
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Built-in hybrid search runtime — BM25F ± LSH ± RRF/weighted fusion,
|
|
3
|
+
* composing {@link parseListQuery} for ordinary filters / pagination.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type { PromptRef } from "../../manifest/types.ts";
|
|
7
|
+
import type { SqlConnection, SqlRow } from "../../drivers/types.ts";
|
|
8
|
+
import { parseListQuery, resolveListScope, type ListOptions } from "./list-query.ts";
|
|
9
|
+
import { bm25fScore, termFrequencies, tokenize } from "./search-bm25.ts";
|
|
10
|
+
import { SearchConfigError, LSH_DEFAULT_K } from "./search-errors.ts";
|
|
11
|
+
import { fuseLists, type FuseOptions } from "./search-fusion.ts";
|
|
12
|
+
import {
|
|
13
|
+
cosineSimilarity,
|
|
14
|
+
deserializePlanes,
|
|
15
|
+
lshBucket,
|
|
16
|
+
lshBucketToSql,
|
|
17
|
+
neighborBuckets,
|
|
18
|
+
} from "./search-lsh.ts";
|
|
19
|
+
import {
|
|
20
|
+
embColumn,
|
|
21
|
+
lshColumn,
|
|
22
|
+
OKE_SEARCH_DF,
|
|
23
|
+
OKE_SEARCH_PLANES,
|
|
24
|
+
OKE_SEARCH_STATS,
|
|
25
|
+
OKE_TSV_COL,
|
|
26
|
+
} from "./search-ddl.ts";
|
|
27
|
+
import { compileWhere } from "./sql-condition.ts";
|
|
28
|
+
import { resolveTableName, type TableHandle } from "./table.ts";
|
|
29
|
+
|
|
30
|
+
/** Search options — `query` / `fuse` / `rerank` are search-specific; rest is list grammar. */
|
|
31
|
+
export interface SqlSearchOptions extends ListOptions {
|
|
32
|
+
/** Hybrid relevance string (BM25 ± semantic). Not the same as list `?search=` LIKE. */
|
|
33
|
+
readonly query: string;
|
|
34
|
+
readonly fuse?: FuseOptions;
|
|
35
|
+
/** Default `false` — never silently invokes fx.ask. */
|
|
36
|
+
readonly rerank?: false | { readonly model: PromptRef };
|
|
37
|
+
/** Extra PostgREST-shaped filter input (same as list / live). */
|
|
38
|
+
readonly filterInput?: unknown;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface SqlSearchResult {
|
|
42
|
+
readonly data: SqlRow[];
|
|
43
|
+
readonly meta: {
|
|
44
|
+
readonly engine: Array<"bm25" | "lsh">;
|
|
45
|
+
readonly fusedBy?: "rrf" | "weighted";
|
|
46
|
+
readonly rrfK?: number;
|
|
47
|
+
readonly limit: number;
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface SearchColumnMeta {
|
|
52
|
+
readonly key: string;
|
|
53
|
+
readonly sqlName: string;
|
|
54
|
+
readonly weight: number;
|
|
55
|
+
readonly embed?: { readonly dims: number; readonly model?: string };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface RunSqlSearchDeps {
|
|
59
|
+
readonly conn: SqlConnection;
|
|
60
|
+
readonly table: TableHandle | unknown;
|
|
61
|
+
readonly columns: readonly SearchColumnMeta[];
|
|
62
|
+
readonly pkSqlName: string;
|
|
63
|
+
readonly options: SqlSearchOptions;
|
|
64
|
+
/** Optional embed for the query string when LSH fields exist. */
|
|
65
|
+
readonly embedQuery?: (text: string, model?: string) => Promise<readonly number[]>;
|
|
66
|
+
/** Optional rerank via fx.ask — only when options.rerank is set. */
|
|
67
|
+
readonly askRerank?: (
|
|
68
|
+
model: PromptRef,
|
|
69
|
+
input: {
|
|
70
|
+
query: string;
|
|
71
|
+
docs: Array<{ id: string; text: string; score: number }>;
|
|
72
|
+
},
|
|
73
|
+
) => Promise<{ rankedIds?: string[] }>;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Execute hybrid search against PostgreSQL using GIN + LSH B-tree candidates,
|
|
78
|
+
* then BM25F / cosine / fusion in-process (correct, testable math).
|
|
79
|
+
*
|
|
80
|
+
* @param deps - Connection, table meta, options
|
|
81
|
+
*/
|
|
82
|
+
export async function runSqlSearch(deps: RunSqlSearchDeps): Promise<SqlSearchResult> {
|
|
83
|
+
const { conn, options, columns, pkSqlName } = deps;
|
|
84
|
+
const tableName = resolveTableName(deps.table);
|
|
85
|
+
if (!tableName) throw new Error("search(): table name could not be resolved");
|
|
86
|
+
|
|
87
|
+
const searchable = columns.filter((c) => c.weight > 0);
|
|
88
|
+
if (searchable.length === 0) {
|
|
89
|
+
throw new SearchConfigError(tableName, "*", "no .searchable() columns on this table");
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const { query, fuse, rerank, filterInput, ...listOpts } = options;
|
|
93
|
+
const scope = resolveListScope(deps.table, listOpts);
|
|
94
|
+
const parsed = parseListQuery(filterInput ?? {}, scope.query);
|
|
95
|
+
if (!parsed.ok) {
|
|
96
|
+
throw parsed.failure instanceof Error
|
|
97
|
+
? parsed.failure
|
|
98
|
+
: new Error(`search(): invalid list filters (${String(parsed.failure)})`);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const limit = parsed.page.limit ?? scope.limit;
|
|
102
|
+
const engines: Array<"bm25" | "lsh"> = ["bm25"];
|
|
103
|
+
const embedFields = searchable.filter((c) => c.embed);
|
|
104
|
+
if (embedFields.length > 0) engines.push("lsh");
|
|
105
|
+
|
|
106
|
+
// Candidate retrieval: GIN tsvector match OR LSH buckets.
|
|
107
|
+
const params: unknown[] = [query];
|
|
108
|
+
let whereSql = `${OKE_TSV_COL} @@ plainto_tsquery('english', ?)`;
|
|
109
|
+
let queryVec: readonly number[] | undefined;
|
|
110
|
+
const bucketParams: bigint[] = [];
|
|
111
|
+
|
|
112
|
+
if (embedFields.length > 0 && deps.embedQuery) {
|
|
113
|
+
const model = embedFields[0]?.embed?.model;
|
|
114
|
+
queryVec = await deps.embedQuery(query, model);
|
|
115
|
+
for (const field of embedFields) {
|
|
116
|
+
const dims = field.embed!.dims;
|
|
117
|
+
if (queryVec.length !== dims) {
|
|
118
|
+
throw new SearchConfigError(
|
|
119
|
+
tableName,
|
|
120
|
+
field.sqlName,
|
|
121
|
+
`query embedding length ${queryVec.length} !== declared dims ${dims}`,
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
const planesRow = await conn.query(
|
|
125
|
+
`SELECT k, planes FROM ${OKE_SEARCH_PLANES} WHERE table_name = ? AND column_name = ?`,
|
|
126
|
+
[tableName, field.sqlName],
|
|
127
|
+
);
|
|
128
|
+
const row = planesRow[0];
|
|
129
|
+
if (!row) {
|
|
130
|
+
throw new SearchConfigError(
|
|
131
|
+
tableName,
|
|
132
|
+
field.sqlName,
|
|
133
|
+
`missing hyperplanes — run oke db search-backfill or ensure push applied search DDL`,
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
const k = Number(row["k"] ?? LSH_DEFAULT_K);
|
|
137
|
+
const planesBuf = row["planes"] as Buffer;
|
|
138
|
+
const planes = deserializePlanes(Buffer.from(planesBuf), k);
|
|
139
|
+
const bucket = lshBucket(queryVec, planes);
|
|
140
|
+
for (const b of neighborBuckets(bucket, k)) {
|
|
141
|
+
bucketParams.push(b);
|
|
142
|
+
}
|
|
143
|
+
whereSql += ` OR ${lshColumn(field.sqlName)} = ANY(?::bigint[])`;
|
|
144
|
+
params.push(`{${bucketParams.map((b) => lshBucketToSql(b)).join(",")}}`);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// Compose list-grammar filters.
|
|
149
|
+
if (parsed.page.where) {
|
|
150
|
+
const compiled = compileWhere(parsed.page.where);
|
|
151
|
+
if (compiled.clause) {
|
|
152
|
+
whereSql = `(${whereSql}) AND (${compiled.clause})`;
|
|
153
|
+
params.push(...compiled.params);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const selectCols = [
|
|
158
|
+
quoteIdent(pkSqlName),
|
|
159
|
+
...searchable.map((c) => quoteIdent(c.sqlName)),
|
|
160
|
+
...embedFields.map((c) => quoteIdent(embColumn(c.sqlName))),
|
|
161
|
+
].join(", ");
|
|
162
|
+
|
|
163
|
+
// Oversample candidates for fusion then truncate.
|
|
164
|
+
const fetchLimit = Math.min(Math.max(limit * 5, 50), 500);
|
|
165
|
+
const candidateSql = `SELECT ${selectCols} FROM ${quoteIdent(tableName)} WHERE ${whereSql} LIMIT ${fetchLimit}`;
|
|
166
|
+
const candidates = await conn.query(candidateSql, params);
|
|
167
|
+
|
|
168
|
+
const statsRows = await conn.query(
|
|
169
|
+
`SELECT n, avgdl FROM ${OKE_SEARCH_STATS} WHERE table_name = ?`,
|
|
170
|
+
[tableName],
|
|
171
|
+
);
|
|
172
|
+
const N = Number(statsRows[0]?.["n"] ?? candidates.length) || candidates.length;
|
|
173
|
+
const avgdl = Number(statsRows[0]?.["avgdl"] ?? 50) || 50;
|
|
174
|
+
|
|
175
|
+
const dfRows = await conn.query(`SELECT term, df FROM ${OKE_SEARCH_DF} WHERE table_name = ?`, [
|
|
176
|
+
tableName,
|
|
177
|
+
]);
|
|
178
|
+
const df = new Map<string, number>();
|
|
179
|
+
for (const r of dfRows) {
|
|
180
|
+
df.set(String(r["term"]), Number(r["df"] ?? 0));
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const qTerms = tokenize(query);
|
|
184
|
+
const bm25Hits: Array<{ id: string; score: number; rank: number; row: SqlRow }> = [];
|
|
185
|
+
const vecHits: Array<{ id: string; score: number; rank: number; row: SqlRow }> = [];
|
|
186
|
+
|
|
187
|
+
for (const row of candidates) {
|
|
188
|
+
const id = String(row[pkSqlName] ?? row["id"] ?? "");
|
|
189
|
+
const fields = searchable.map((c) => {
|
|
190
|
+
const text = String(row[c.sqlName] ?? "");
|
|
191
|
+
return { weight: c.weight, tf: termFrequencies(tokenize(text)) };
|
|
192
|
+
});
|
|
193
|
+
let docLen = 0;
|
|
194
|
+
for (const f of fields) {
|
|
195
|
+
for (const n of f.tf.values()) docLen += n;
|
|
196
|
+
}
|
|
197
|
+
const score = bm25fScore(qTerms, fields, docLen, avgdl, N, df);
|
|
198
|
+
bm25Hits.push({ id, score, rank: 0, row });
|
|
199
|
+
}
|
|
200
|
+
bm25Hits.sort((a, b) => b.score - a.score);
|
|
201
|
+
bm25Hits.forEach((h, i) => {
|
|
202
|
+
h.rank = i + 1;
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
if (queryVec && embedFields.length > 0) {
|
|
206
|
+
for (const row of candidates) {
|
|
207
|
+
const id = String(row[pkSqlName] ?? row["id"] ?? "");
|
|
208
|
+
let best = -1;
|
|
209
|
+
for (const field of embedFields) {
|
|
210
|
+
const emb = parseRealArray(row[embColumn(field.sqlName)]);
|
|
211
|
+
if (!emb) continue;
|
|
212
|
+
if (emb.length !== field.embed!.dims) {
|
|
213
|
+
throw new SearchConfigError(
|
|
214
|
+
tableName,
|
|
215
|
+
field.sqlName,
|
|
216
|
+
`stored embedding length ${emb.length} !== declared dims ${field.embed!.dims}`,
|
|
217
|
+
);
|
|
218
|
+
}
|
|
219
|
+
const sim = cosineSimilarity(queryVec, emb);
|
|
220
|
+
if (sim > best) best = sim;
|
|
221
|
+
}
|
|
222
|
+
if (best >= 0) vecHits.push({ id, score: best, rank: 0, row });
|
|
223
|
+
}
|
|
224
|
+
vecHits.sort((a, b) => b.score - a.score);
|
|
225
|
+
vecHits.forEach((h, i) => {
|
|
226
|
+
h.rank = i + 1;
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
const rowById = new Map<string, SqlRow>();
|
|
231
|
+
for (const h of bm25Hits) rowById.set(h.id, h.row);
|
|
232
|
+
for (const h of vecHits) rowById.set(h.id, h.row);
|
|
233
|
+
|
|
234
|
+
let orderedIds: string[];
|
|
235
|
+
let fusedBy: "rrf" | "weighted" | undefined;
|
|
236
|
+
let rrfK: number | undefined;
|
|
237
|
+
|
|
238
|
+
if (vecHits.length > 0) {
|
|
239
|
+
const lists = new Map([
|
|
240
|
+
["bm25", bm25Hits.map(({ id, score, rank }) => ({ id, score, rank }))],
|
|
241
|
+
["vector", vecHits.map(({ id, score, rank }) => ({ id, score, rank }))],
|
|
242
|
+
]);
|
|
243
|
+
const fused = fuseLists(lists, fuse);
|
|
244
|
+
orderedIds = fused.hits.map((h) => h.id);
|
|
245
|
+
fusedBy = fused.strategy;
|
|
246
|
+
rrfK = fused.k;
|
|
247
|
+
} else {
|
|
248
|
+
orderedIds = bm25Hits.map((h) => h.id);
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
let data = orderedIds
|
|
252
|
+
.slice(0, limit)
|
|
253
|
+
.map((id) => rowById.get(id))
|
|
254
|
+
.filter((r): r is SqlRow => r !== undefined);
|
|
255
|
+
|
|
256
|
+
if (rerank && typeof rerank === "object" && rerank.model && deps.askRerank) {
|
|
257
|
+
const docs = data.map((row) => {
|
|
258
|
+
const id = String(row[pkSqlName] ?? row["id"] ?? "");
|
|
259
|
+
const text = searchable.map((c) => String(row[c.sqlName] ?? "")).join("\n");
|
|
260
|
+
return { id, text, score: 0 };
|
|
261
|
+
});
|
|
262
|
+
const out = await deps.askRerank(rerank.model, { query, docs });
|
|
263
|
+
if (out.rankedIds && out.rankedIds.length > 0) {
|
|
264
|
+
const byId = new Map(data.map((r) => [String(r[pkSqlName] ?? r["id"]), r]));
|
|
265
|
+
data = out.rankedIds.map((id) => byId.get(id)).filter((r): r is SqlRow => r !== undefined);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
return {
|
|
270
|
+
data,
|
|
271
|
+
meta: {
|
|
272
|
+
engine: engines,
|
|
273
|
+
...(fusedBy ? { fusedBy } : {}),
|
|
274
|
+
...(rrfK !== undefined ? { rrfK } : {}),
|
|
275
|
+
limit,
|
|
276
|
+
},
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
function quoteIdent(name: string): string {
|
|
281
|
+
if (!/^[a-z_][a-z0-9_]*$/i.test(name)) {
|
|
282
|
+
return `"${name.replaceAll('"', '""')}"`;
|
|
283
|
+
}
|
|
284
|
+
return name;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* Coerce a Postgres `real[]` driver value into a number array.
|
|
289
|
+
*
|
|
290
|
+
* @param value - Array or `{1,2,3}` text
|
|
291
|
+
*/
|
|
292
|
+
function parseRealArray(value: unknown): number[] | undefined {
|
|
293
|
+
if (Array.isArray(value)) return value.map((x) => Number(x));
|
|
294
|
+
if (typeof value === "string") {
|
|
295
|
+
const inner = value.trim().replace(/^\{/, "").replace(/\}$/, "");
|
|
296
|
+
if (inner.length === 0) return [];
|
|
297
|
+
return inner.split(",").map((x) => Number(x.trim()));
|
|
298
|
+
}
|
|
299
|
+
return undefined;
|
|
300
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Built-in hybrid search — unit correctness (BM25F, LSH, RRF, searchable API).
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { describe, expect, test } from "bun:test";
|
|
6
|
+
import { field, store } from "../store.ts";
|
|
7
|
+
import { bm25fScore, termFrequencies, tokenize } from "./search-bm25.ts";
|
|
8
|
+
import { fuseLists, fuseRrf, fuseWeighted } from "./search-fusion.ts";
|
|
9
|
+
import { RRF_DEFAULT_K, SearchConfigError } from "./search-errors.ts";
|
|
10
|
+
import {
|
|
11
|
+
cosineSimilarity,
|
|
12
|
+
generateHyperplanes,
|
|
13
|
+
hyperplaneSeed,
|
|
14
|
+
lshBucket,
|
|
15
|
+
lshBucketFromSql,
|
|
16
|
+
lshBucketToSql,
|
|
17
|
+
neighborBuckets,
|
|
18
|
+
} from "./search-lsh.ts";
|
|
19
|
+
|
|
20
|
+
describe("field.searchable / .embed", () => {
|
|
21
|
+
test("searchable weight defaults to 1; embed requires prior searchable", () => {
|
|
22
|
+
const articles = store.schema.table("articles", {
|
|
23
|
+
id: field.text().primaryKey(),
|
|
24
|
+
title: field.text().searchable({ weight: 2 }).notNull(),
|
|
25
|
+
body: field.text().searchable().embed({ dims: 8, model: "embedder" }),
|
|
26
|
+
});
|
|
27
|
+
expect(articles.columns.title.search).toEqual({ searchable: true, weight: 2 });
|
|
28
|
+
expect(articles.columns.body.search?.embed).toEqual({ dims: 8, model: "embedder" });
|
|
29
|
+
expect(() => field.text().embed({ dims: 8 })).toThrow(/prior \.searchable/);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
test("bare .embed() and partial options are allowed at declare time", () => {
|
|
33
|
+
const bare = field.text().searchable().embed();
|
|
34
|
+
const modelOnly = field.text().searchable().embed({ model: "embedder" });
|
|
35
|
+
expect(bare).toBeDefined();
|
|
36
|
+
expect(modelOnly).toBeDefined();
|
|
37
|
+
const table = store.schema.table("articles", {
|
|
38
|
+
id: field.text().primaryKey(),
|
|
39
|
+
body: field.text().searchable().embed(),
|
|
40
|
+
caption: field.text().searchable().embed({ model: "other" }),
|
|
41
|
+
});
|
|
42
|
+
expect(table.columns.body.search?.embed).toEqual({});
|
|
43
|
+
expect(table.columns.caption.search?.embed).toEqual({ model: "other" });
|
|
44
|
+
});
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
describe("BM25F", () => {
|
|
48
|
+
test("field weight applied before saturation matches hand reference", () => {
|
|
49
|
+
const query = ["cat"];
|
|
50
|
+
const df = new Map([["cat", 1]]);
|
|
51
|
+
const N = 2;
|
|
52
|
+
const avgdl = 2;
|
|
53
|
+
// Doc A: title has cat once (weight 2); Doc B: body has cat once (weight 1)
|
|
54
|
+
const scoreA = bm25fScore(
|
|
55
|
+
query,
|
|
56
|
+
[{ weight: 2, tf: termFrequencies(["cat"]) }],
|
|
57
|
+
1,
|
|
58
|
+
avgdl,
|
|
59
|
+
N,
|
|
60
|
+
df,
|
|
61
|
+
);
|
|
62
|
+
const scoreB = bm25fScore(
|
|
63
|
+
query,
|
|
64
|
+
[{ weight: 1, tf: termFrequencies(["cat"]) }],
|
|
65
|
+
1,
|
|
66
|
+
avgdl,
|
|
67
|
+
N,
|
|
68
|
+
df,
|
|
69
|
+
);
|
|
70
|
+
expect(scoreA).toBeGreaterThan(scoreB);
|
|
71
|
+
// Hand: weightedTf=2 vs 1 with same IDF → A ranks higher
|
|
72
|
+
expect(tokenize("Hello, CAT!")).toEqual(["hello", "cat"]);
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
describe("LSH", () => {
|
|
77
|
+
test("planes are stable for the same seed; near vectors collide", () => {
|
|
78
|
+
const seed = hyperplaneSeed("articles", "body", 8, 16);
|
|
79
|
+
const a = generateHyperplanes(seed, 8, 16);
|
|
80
|
+
const b = generateHyperplanes(seed, 8, 16);
|
|
81
|
+
expect(a[0]![0]).toBe(b[0]![0]);
|
|
82
|
+
const v = [1, 0, 0, 0, 0, 0, 0, 0];
|
|
83
|
+
const near = [0.99, 0.01, 0, 0, 0, 0, 0, 0];
|
|
84
|
+
const ba = lshBucket(v, a);
|
|
85
|
+
const bn = lshBucket(near, a);
|
|
86
|
+
// Exact or Hamming-1 neighbor should include near for this orientation
|
|
87
|
+
const neighbors = new Set(neighborBuckets(ba, 16).map(String));
|
|
88
|
+
expect(neighbors.has(String(bn)) || ba === bn).toBe(true);
|
|
89
|
+
expect(cosineSimilarity(v, near)).toBeGreaterThan(0.9);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
test("dimension mismatch fails loud", () => {
|
|
93
|
+
const planes = generateHyperplanes("s", 4, 8);
|
|
94
|
+
expect(() => lshBucket([1, 2], planes)).toThrow(/length/);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
test("lshBucketToSql round-trips high bit into signed bigint range", () => {
|
|
98
|
+
const high = 1n << 63n;
|
|
99
|
+
const sql = lshBucketToSql(high);
|
|
100
|
+
expect(BigInt(sql) < 0n).toBe(true);
|
|
101
|
+
expect(lshBucketFromSql(sql)).toBe(high);
|
|
102
|
+
expect(lshBucketToSql(42n)).toBe("42");
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
describe("fusion", () => {
|
|
107
|
+
test("RRF default k=60 matches Cormack hand ranks", () => {
|
|
108
|
+
expect(RRF_DEFAULT_K).toBe(60);
|
|
109
|
+
const lists = new Map([
|
|
110
|
+
[
|
|
111
|
+
"bm25",
|
|
112
|
+
[
|
|
113
|
+
{ id: "a", score: 10, rank: 1 },
|
|
114
|
+
{ id: "b", score: 5, rank: 2 },
|
|
115
|
+
],
|
|
116
|
+
],
|
|
117
|
+
[
|
|
118
|
+
"vector",
|
|
119
|
+
[
|
|
120
|
+
{ id: "b", score: 0.9, rank: 1 },
|
|
121
|
+
{ id: "a", score: 0.5, rank: 2 },
|
|
122
|
+
],
|
|
123
|
+
],
|
|
124
|
+
]);
|
|
125
|
+
const hits = fuseRrf(lists, 60);
|
|
126
|
+
// a: 1/61 + 1/62; b: 1/62 + 1/61 — equal contributions swapped → tied-ish; b wins on equal?
|
|
127
|
+
// a = 1/61 + 1/62; b = 1/62 + 1/61 — identical
|
|
128
|
+
expect(hits[0]!.score).toBeCloseTo(hits[1]!.score, 10);
|
|
129
|
+
const weighted = fuseWeighted(lists, { bm25: 1, vector: 0 });
|
|
130
|
+
expect(weighted[0]!.id).toBe("a");
|
|
131
|
+
const fused = fuseLists(lists);
|
|
132
|
+
expect(fused.strategy).toBe("rrf");
|
|
133
|
+
expect(fused.k).toBe(60);
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
describe("SearchConfigError", () => {
|
|
138
|
+
test("names table and column", () => {
|
|
139
|
+
const err = new SearchConfigError("notes", "body", "missing ai");
|
|
140
|
+
expect(err.name).toBe("SearchConfigError");
|
|
141
|
+
expect(err.table).toBe("notes");
|
|
142
|
+
expect(err.message).toContain("notes.body");
|
|
143
|
+
});
|
|
144
|
+
});
|
|
@@ -19,8 +19,8 @@ import { maskRows, tableFromSql } from "./classify.ts";
|
|
|
19
19
|
import { isMissingDomainRelationError } from "./missing-relation.ts";
|
|
20
20
|
import {
|
|
21
21
|
andWhere,
|
|
22
|
-
compileOrderBy,
|
|
23
22
|
compileWhere,
|
|
23
|
+
compileOrderBy,
|
|
24
24
|
resolveSelectColumns,
|
|
25
25
|
type WhereMap,
|
|
26
26
|
} from "./sql-condition.ts";
|
|
@@ -32,6 +32,9 @@ import {
|
|
|
32
32
|
resolveTableName,
|
|
33
33
|
type TableHandle,
|
|
34
34
|
} from "./table.ts";
|
|
35
|
+
import { isSchemaTableDecl } from "./schema-decl.ts";
|
|
36
|
+
import { SearchConfigError } from "./search-errors.ts";
|
|
37
|
+
import { runSqlSearch, type SearchColumnMeta, type SqlSearchOptions } from "./search-runtime.ts";
|
|
35
38
|
|
|
36
39
|
export type { WhereMap } from "./sql-condition.ts";
|
|
37
40
|
|
|
@@ -204,6 +207,11 @@ export interface SqlSessionOptions {
|
|
|
204
207
|
readonly domainDdl?: DomainDdlMode;
|
|
205
208
|
/** Gate identity — stamps `oke.*` on postgres / pglite. */
|
|
206
209
|
readonly rls?: RlsIdentity;
|
|
210
|
+
/**
|
|
211
|
+
* Optional query-time embedding for hybrid LSH (wired from `fx.embed` at boot).
|
|
212
|
+
* When omitted, search is BM25-only even if columns declare `.embed()`.
|
|
213
|
+
*/
|
|
214
|
+
readonly embedQuery?: (text: string, model?: string) => Promise<readonly number[]>;
|
|
207
215
|
}
|
|
208
216
|
|
|
209
217
|
/**
|
|
@@ -363,6 +371,11 @@ export interface SqlStoreHandle {
|
|
|
363
371
|
readonly routedRole: "primary" | "replica";
|
|
364
372
|
/** Underlying driver id. */
|
|
365
373
|
readonly driverId: SqlConnection["driverId"];
|
|
374
|
+
/**
|
|
375
|
+
* Egress identity when the SQL connection leaves the process.
|
|
376
|
+
* Omitted for in-process drivers (`pglite`, `memory`).
|
|
377
|
+
*/
|
|
378
|
+
readonly external?: SqlConnection["external"];
|
|
366
379
|
/**
|
|
367
380
|
* Start a select. No-arg form infers the row from `.from(table)`.
|
|
368
381
|
* A column map (`select({ clicks: links.clicks })`) stays {@link SqlRow}.
|
|
@@ -453,6 +466,18 @@ export interface SqlStoreHandle {
|
|
|
453
466
|
* @param options - Page options
|
|
454
467
|
*/
|
|
455
468
|
page(table: TableHandle | unknown, options: SqlPageOptions): Promise<SqlRow[]>;
|
|
469
|
+
/**
|
|
470
|
+
* Built-in hybrid search (BM25 ± LSH ± fusion) on `.searchable()` columns.
|
|
471
|
+
* Reuses {@link parseListQuery} for filters / limit / cursor; `query` is
|
|
472
|
+
* the relevance string (not list-grammar `?search=` LIKE).
|
|
473
|
+
*
|
|
474
|
+
* @param table - Table with searchable columns
|
|
475
|
+
* @param options - query + list filters + optional fuse / rerank
|
|
476
|
+
*/
|
|
477
|
+
search(
|
|
478
|
+
table: TableHandle | unknown,
|
|
479
|
+
options: import("./search-runtime.ts").SqlSearchOptions,
|
|
480
|
+
): Promise<import("./search-runtime.ts").SqlSearchResult>;
|
|
456
481
|
/**
|
|
457
482
|
* Ensure a simple table exists (test / bootstrap helper).
|
|
458
483
|
*
|
|
@@ -507,7 +532,7 @@ export function createSqlStoreHandle(
|
|
|
507
532
|
ref: `sql:${string}`,
|
|
508
533
|
options: SqlSessionOptions,
|
|
509
534
|
): SqlStoreHandle {
|
|
510
|
-
const { connection, classifications, revealPii, rls } = options;
|
|
535
|
+
const { connection, classifications, revealPii, rls, embedQuery } = options;
|
|
511
536
|
const domainDdl: DomainDdlMode = options.domainDdl ?? "ensure";
|
|
512
537
|
|
|
513
538
|
function mask(rows: SqlRow[], table?: string): SqlRow[] {
|
|
@@ -515,7 +540,7 @@ export function createSqlStoreHandle(
|
|
|
515
540
|
}
|
|
516
541
|
|
|
517
542
|
/**
|
|
518
|
-
* Run a connection op; remap missing-relation driver errors to
|
|
543
|
+
* Run a connection op; remap missing-relation driver errors to OKE1110
|
|
519
544
|
* when domain auto-DDL is off.
|
|
520
545
|
*/
|
|
521
546
|
async function withSchemaGuard<T>(fn: () => Promise<T>): Promise<T> {
|
|
@@ -744,6 +769,7 @@ export function createSqlStoreHandle(
|
|
|
744
769
|
ref,
|
|
745
770
|
routedRole: options.routedRole,
|
|
746
771
|
driverId: connection.driverId,
|
|
772
|
+
...(connection.external !== undefined ? { external: connection.external } : {}),
|
|
747
773
|
|
|
748
774
|
select(columns?: unknown) {
|
|
749
775
|
return {
|
|
@@ -1134,6 +1160,20 @@ export function createSqlStoreHandle(
|
|
|
1134
1160
|
});
|
|
1135
1161
|
},
|
|
1136
1162
|
|
|
1163
|
+
async search(table: TableHandle | unknown, searchOptions: SqlSearchOptions) {
|
|
1164
|
+
await ensureFromMeta(table);
|
|
1165
|
+
const columns = searchColumnsFromTable(table);
|
|
1166
|
+
const pk = resolvePkColumn(table);
|
|
1167
|
+
return runSqlSearch({
|
|
1168
|
+
conn: connection,
|
|
1169
|
+
table,
|
|
1170
|
+
columns,
|
|
1171
|
+
pkSqlName: pk,
|
|
1172
|
+
options: searchOptions,
|
|
1173
|
+
...(embedQuery ? { embedQuery } : {}),
|
|
1174
|
+
});
|
|
1175
|
+
},
|
|
1176
|
+
|
|
1137
1177
|
async ensureTable(table: TableHandle) {
|
|
1138
1178
|
const cols = Object.values(table.columns);
|
|
1139
1179
|
const pk = resolvePkColumn(table);
|
|
@@ -1169,6 +1209,46 @@ export function resolvePkColumn(table: unknown): string {
|
|
|
1169
1209
|
return "id";
|
|
1170
1210
|
}
|
|
1171
1211
|
|
|
1212
|
+
/**
|
|
1213
|
+
* Collect `.searchable()` / `.embed()` column meta from a schema table.
|
|
1214
|
+
*
|
|
1215
|
+
* @param table - Table handle
|
|
1216
|
+
*/
|
|
1217
|
+
function searchColumnsFromTable(table: unknown): SearchColumnMeta[] {
|
|
1218
|
+
if (!isSchemaTableDecl(table)) {
|
|
1219
|
+
throw new Error(
|
|
1220
|
+
"search(): table must be a store.schema.table() declaration with .searchable() columns",
|
|
1221
|
+
);
|
|
1222
|
+
}
|
|
1223
|
+
const out: SearchColumnMeta[] = [];
|
|
1224
|
+
const tableName = resolveTableName(table);
|
|
1225
|
+
for (const [key, col] of Object.entries(table.columns)) {
|
|
1226
|
+
if (!col.search?.searchable) continue;
|
|
1227
|
+
let embed: SearchColumnMeta["embed"];
|
|
1228
|
+
if (col.search.embed) {
|
|
1229
|
+
const dims = col.search.embed.dims;
|
|
1230
|
+
if (dims === undefined) {
|
|
1231
|
+
throw new SearchConfigError(
|
|
1232
|
+
tableName,
|
|
1233
|
+
key,
|
|
1234
|
+
`.embed() needs dims — set oke({ store: { search: { embed: { model, dims } } } }) or pass dims on .embed({ dims })`,
|
|
1235
|
+
);
|
|
1236
|
+
}
|
|
1237
|
+
embed = {
|
|
1238
|
+
dims,
|
|
1239
|
+
...(col.search.embed.model ? { model: col.search.embed.model } : {}),
|
|
1240
|
+
};
|
|
1241
|
+
}
|
|
1242
|
+
out.push({
|
|
1243
|
+
key,
|
|
1244
|
+
sqlName: col.sqlName,
|
|
1245
|
+
weight: col.search.weight,
|
|
1246
|
+
...(embed ? { embed } : {}),
|
|
1247
|
+
});
|
|
1248
|
+
}
|
|
1249
|
+
return out;
|
|
1250
|
+
}
|
|
1251
|
+
|
|
1172
1252
|
function asNumber(value: unknown, column: string): number {
|
|
1173
1253
|
if (typeof value === "number") return value;
|
|
1174
1254
|
if (typeof value === "bigint") return Number(value);
|
|
@@ -293,6 +293,27 @@ function drizzleSqlType(
|
|
|
293
293
|
return "TEXT";
|
|
294
294
|
}
|
|
295
295
|
|
|
296
|
+
/**
|
|
297
|
+
* Bind value for a temporal SQL column. Epoch-ms numbers (e.g. `fx.clock.now()`)
|
|
298
|
+
* become `Date` so Postgres `timestamp` / `date` accept them; other values pass through.
|
|
299
|
+
*
|
|
300
|
+
* @param sqlType - Resolved DDL type
|
|
301
|
+
* @param value - Incoming JS value
|
|
302
|
+
*/
|
|
303
|
+
export function coerceTemporalBindValue(
|
|
304
|
+
sqlType: ResolvedColumn["sqlType"],
|
|
305
|
+
value: unknown,
|
|
306
|
+
): unknown {
|
|
307
|
+
if (
|
|
308
|
+
(sqlType === "TIMESTAMP" || sqlType === "DATE") &&
|
|
309
|
+
typeof value === "number" &&
|
|
310
|
+
Number.isFinite(value)
|
|
311
|
+
) {
|
|
312
|
+
return new Date(value);
|
|
313
|
+
}
|
|
314
|
+
return value;
|
|
315
|
+
}
|
|
316
|
+
|
|
296
317
|
/**
|
|
297
318
|
* Map a JS-keyed row to SQL column names; apply Drizzle `$defaultFn`s.
|
|
298
319
|
*
|
|
@@ -315,7 +336,7 @@ export function prepareInsertRow(
|
|
|
315
336
|
else if (col.sqlName in row) value = row[col.sqlName];
|
|
316
337
|
else if (col.defaultFn) value = col.defaultFn();
|
|
317
338
|
else continue;
|
|
318
|
-
out[col.sqlName] = value;
|
|
339
|
+
out[col.sqlName] = coerceTemporalBindValue(col.sqlType, value);
|
|
319
340
|
}
|
|
320
341
|
// Pass through unknown keys as-is (already SQL names).
|
|
321
342
|
for (const [k, v] of Object.entries(row)) {
|
|
@@ -344,7 +365,7 @@ export function prepareUpdateRow(
|
|
|
344
365
|
const out: Record<string, unknown> = {};
|
|
345
366
|
for (const [k, v] of Object.entries(row)) {
|
|
346
367
|
const col = byKey.get(k) ?? bySql.get(k);
|
|
347
|
-
out[col?.sqlName ?? k] = v;
|
|
368
|
+
out[col?.sqlName ?? k] = col ? coerceTemporalBindValue(col.sqlType, v) : v;
|
|
348
369
|
}
|
|
349
370
|
return out;
|
|
350
371
|
}
|
|
@@ -45,10 +45,8 @@ describe("fx.store(db).upsert via app.fetch (real boot)", () => {
|
|
|
45
45
|
const db = store.sql("app", { schema: { notes } });
|
|
46
46
|
|
|
47
47
|
const seed = on(
|
|
48
|
-
http.post("/seed").public(),
|
|
48
|
+
http.post("/seed", { in: UpsertIn, out: UpsertOut }).public(),
|
|
49
49
|
flow("notes.seed", {
|
|
50
|
-
in: UpsertIn,
|
|
51
|
-
out: UpsertOut,
|
|
52
50
|
effects: { writes: ["sql:app"], reads: ["sql:app"] },
|
|
53
51
|
do: async (input, fx) =>
|
|
54
52
|
fx
|