claude-smart 0.2.48 → 0.2.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +6 -2
- package/bin/claude-smart.js +289 -56
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/.codex-plugin/plugin.json +1 -1
- package/plugin/opencode/dist/server.mjs +60 -2
- package/plugin/opencode/server.mts +64 -2
- package/plugin/pyproject.toml +2 -2
- package/plugin/scripts/_lib.sh +58 -6
- package/plugin/scripts/backend-service.sh +15 -10
- package/plugin/scripts/codex-hook.js +16 -4
- package/plugin/scripts/dashboard-service.sh +1 -1
- package/plugin/scripts/ensure-plugin-root.sh +106 -8
- package/plugin/scripts/opencode-claude-compat.js +8 -1
- package/plugin/scripts/smart-install.sh +101 -20
- package/plugin/src/README.md +1 -1
- package/plugin/src/claude_smart/cli.py +79 -29
- package/plugin/src/claude_smart/env_config.py +53 -11
- package/plugin/uv.lock +5 -5
- package/plugin/vendor/reflexio/reflexio/cli/README.md +3 -0
- package/plugin/vendor/reflexio/reflexio/client/client.py +40 -6
- package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/entities.py +18 -0
- package/plugin/vendor/reflexio/reflexio/server/api.py +15 -4
- package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_text_generation.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/llm/litellm_client.py +1 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/local_embedding_provider.py +183 -1
- package/plugin/vendor/reflexio/reflexio/server/routes/interactions.py +63 -2
- package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/__init__.py +13 -0
- package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/scheduler.py +142 -0
- package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/worker.py +193 -0
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_scheduler.py +1 -3
- package/plugin/vendor/reflexio/reflexio/server/services/generation_service.py +150 -37
- package/plugin/vendor/reflexio/reflexio/server/services/profile/service.py +44 -22
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/__init__.py +2 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_base.py +140 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py +379 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_lineage.py +11 -5
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_requests.py +8 -3
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/base/_fts_vec.py +86 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_agent.py +16 -7
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_source_linkage.py +8 -3
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_user.py +12 -5
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_interaction_store.py +52 -7
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_profile_store.py +28 -4
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/__init__.py +17 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_commit_scope.py +9 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_learning_jobs.py +219 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_interaction_store.py +23 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_profile_store.py +22 -0
|
@@ -7,6 +7,7 @@ are available.
|
|
|
7
7
|
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
|
+
import contextlib
|
|
10
11
|
import functools
|
|
11
12
|
import json
|
|
12
13
|
import logging
|
|
@@ -14,7 +15,7 @@ import math
|
|
|
14
15
|
import re
|
|
15
16
|
import sqlite3
|
|
16
17
|
import threading
|
|
17
|
-
from collections.abc import Callable, Sequence
|
|
18
|
+
from collections.abc import Callable, Generator, Sequence
|
|
18
19
|
from datetime import UTC, datetime
|
|
19
20
|
from pathlib import Path
|
|
20
21
|
from typing import Any, ClassVar, Literal
|
|
@@ -590,6 +591,7 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
|
|
|
590
591
|
# FTS/vec index helpers provided by SQLiteFtsVecMixin via the composed
|
|
591
592
|
# SQLiteStorage MRO; declared here for _migrate_vec_tables's benefit.
|
|
592
593
|
_vec_upsert: Any
|
|
594
|
+
_flush_index_op: Any
|
|
593
595
|
|
|
594
596
|
@staticmethod
|
|
595
597
|
def handle_exceptions(func: Callable[..., Any]) -> Callable[..., Any]:
|
|
@@ -633,6 +635,10 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
|
|
|
633
635
|
|
|
634
636
|
self.db_path = db_path
|
|
635
637
|
self._lock = threading.RLock()
|
|
638
|
+
self._scope_depth = 0
|
|
639
|
+
self._deferred_index_ops: list[
|
|
640
|
+
tuple[str, Any]
|
|
641
|
+
] = [] # (kind, args) flushed post-commit
|
|
636
642
|
|
|
637
643
|
logger.info("SQLite Storage for org %s using db_path: %s", org_id, db_path)
|
|
638
644
|
|
|
@@ -670,6 +676,45 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
|
|
|
670
676
|
# Create tables
|
|
671
677
|
self.migrate()
|
|
672
678
|
|
|
679
|
+
# ------------------------------------------------------------------
|
|
680
|
+
# Transaction scope
|
|
681
|
+
# ------------------------------------------------------------------
|
|
682
|
+
|
|
683
|
+
def _own_transaction(self) -> bool:
|
|
684
|
+
"""True when the caller is NOT inside a commit_scope (owns its BEGIN/commit)."""
|
|
685
|
+
return self._scope_depth == 0
|
|
686
|
+
|
|
687
|
+
@contextlib.contextmanager
|
|
688
|
+
def commit_scope(self) -> Generator[None, None, None]:
|
|
689
|
+
"""Group writes into one atomic commit; defer FTS/vec index ops.
|
|
690
|
+
|
|
691
|
+
Nested scopes join the outermost: only the outer scope commits. On
|
|
692
|
+
exception the whole transaction rolls back and deferred index ops
|
|
693
|
+
are discarded.
|
|
694
|
+
"""
|
|
695
|
+
with self._lock:
|
|
696
|
+
if self._scope_depth > 0:
|
|
697
|
+
self._scope_depth += 1
|
|
698
|
+
try:
|
|
699
|
+
yield
|
|
700
|
+
finally:
|
|
701
|
+
self._scope_depth -= 1
|
|
702
|
+
return
|
|
703
|
+
self.conn.execute("BEGIN IMMEDIATE")
|
|
704
|
+
self._scope_depth = 1
|
|
705
|
+
try:
|
|
706
|
+
yield
|
|
707
|
+
self.conn.commit()
|
|
708
|
+
ops, self._deferred_index_ops = self._deferred_index_ops, []
|
|
709
|
+
for kind, args in ops:
|
|
710
|
+
self._flush_index_op(kind, args) # self-commits — fine post-commit
|
|
711
|
+
except Exception:
|
|
712
|
+
self.conn.rollback()
|
|
713
|
+
self._deferred_index_ops = []
|
|
714
|
+
raise
|
|
715
|
+
finally:
|
|
716
|
+
self._scope_depth = 0
|
|
717
|
+
|
|
673
718
|
# ------------------------------------------------------------------
|
|
674
719
|
# DDL / migration
|
|
675
720
|
# ------------------------------------------------------------------
|
|
@@ -714,6 +759,7 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
|
|
|
714
759
|
self._migrate_retire_profile_change_logs()
|
|
715
760
|
self._migrate_retire_playbook_aggregation_change_logs()
|
|
716
761
|
init_stall_state_table(self.conn)
|
|
762
|
+
self._migrate_learning_jobs()
|
|
717
763
|
return True
|
|
718
764
|
|
|
719
765
|
def _try_load_sqlite_vec(self) -> bool:
|
|
@@ -1261,6 +1307,70 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
|
|
|
1261
1307
|
)
|
|
1262
1308
|
self.conn.commit()
|
|
1263
1309
|
|
|
1310
|
+
def _migrate_learning_jobs(self) -> None:
|
|
1311
|
+
"""Create the learning_jobs table + partial indexes if missing (idempotent).
|
|
1312
|
+
|
|
1313
|
+
Runs ``CREATE TABLE IF NOT EXISTS`` and ``CREATE INDEX IF NOT EXISTS`` only —
|
|
1314
|
+
both are no-ops when the table / indexes already exist. New columns added in
|
|
1315
|
+
subsequent tasks are backfilled via ``PRAGMA table_info`` + ``ALTER TABLE … ADD
|
|
1316
|
+
COLUMN`` (mirroring ``_migrate_lineage_event_table``), because
|
|
1317
|
+
``CREATE TABLE IF NOT EXISTS`` silently skips the DDL on existing databases.
|
|
1318
|
+
|
|
1319
|
+
Called at the end of migrate() so the table is always present on startup.
|
|
1320
|
+
"""
|
|
1321
|
+
with self._lock:
|
|
1322
|
+
self.conn.executescript("""
|
|
1323
|
+
CREATE TABLE IF NOT EXISTS learning_jobs (
|
|
1324
|
+
job_id TEXT PRIMARY KEY,
|
|
1325
|
+
org_id TEXT NOT NULL,
|
|
1326
|
+
user_id TEXT NOT NULL,
|
|
1327
|
+
job_type TEXT NOT NULL DEFAULT 'learning',
|
|
1328
|
+
latest_request_id TEXT,
|
|
1329
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
1330
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
1331
|
+
max_attempts INTEGER NOT NULL DEFAULT 3,
|
|
1332
|
+
claimed_by TEXT,
|
|
1333
|
+
claim_token TEXT,
|
|
1334
|
+
claim_expires_at TEXT,
|
|
1335
|
+
covers_through TEXT,
|
|
1336
|
+
force_extraction INTEGER NOT NULL DEFAULT 0,
|
|
1337
|
+
skip_aggregation INTEGER NOT NULL DEFAULT 0,
|
|
1338
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
|
1339
|
+
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
1340
|
+
);
|
|
1341
|
+
CREATE UNIQUE INDEX IF NOT EXISTS learning_jobs_coalesce
|
|
1342
|
+
ON learning_jobs (org_id, user_id, job_type) WHERE status = 'pending';
|
|
1343
|
+
-- Recreate the poll index with 'failed' in the predicate. CREATE
|
|
1344
|
+
-- INDEX IF NOT EXISTS is a no-op on an existing index with the old
|
|
1345
|
+
-- ('pending','claimed') predicate, so DROP first to migrate it.
|
|
1346
|
+
DROP INDEX IF EXISTS learning_jobs_poll;
|
|
1347
|
+
CREATE INDEX IF NOT EXISTS learning_jobs_poll
|
|
1348
|
+
ON learning_jobs (created_at) WHERE status IN ('pending','failed','claimed');
|
|
1349
|
+
""")
|
|
1350
|
+
# Backfill columns added after the initial release (existing DBs skip
|
|
1351
|
+
# CREATE TABLE IF NOT EXISTS so these must be applied separately).
|
|
1352
|
+
existing_cols = {
|
|
1353
|
+
row["name"]
|
|
1354
|
+
for row in self.conn.execute(
|
|
1355
|
+
"PRAGMA table_info(learning_jobs)"
|
|
1356
|
+
).fetchall()
|
|
1357
|
+
}
|
|
1358
|
+
if "force_extraction" not in existing_cols:
|
|
1359
|
+
self.conn.execute(
|
|
1360
|
+
"ALTER TABLE learning_jobs ADD COLUMN force_extraction INTEGER NOT NULL DEFAULT 0"
|
|
1361
|
+
)
|
|
1362
|
+
if "skip_aggregation" not in existing_cols:
|
|
1363
|
+
self.conn.execute(
|
|
1364
|
+
"ALTER TABLE learning_jobs ADD COLUMN skip_aggregation INTEGER NOT NULL DEFAULT 0"
|
|
1365
|
+
)
|
|
1366
|
+
self.conn.commit()
|
|
1367
|
+
|
|
1368
|
+
def learning_jobs_columns(self) -> list[str]:
|
|
1369
|
+
"""Return the column names of the learning_jobs table."""
|
|
1370
|
+
with self._lock:
|
|
1371
|
+
rows = self.conn.execute("PRAGMA table_info(learning_jobs)").fetchall()
|
|
1372
|
+
return [row["name"] for row in rows]
|
|
1373
|
+
|
|
1264
1374
|
def _migrate_agent_playbook_source_windows(self) -> None:
|
|
1265
1375
|
"""Add source window snapshots to existing agent source mappings."""
|
|
1266
1376
|
cols = {
|
|
@@ -1451,7 +1561,8 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
|
|
|
1451
1561
|
) -> sqlite3.Cursor:
|
|
1452
1562
|
with self._lock:
|
|
1453
1563
|
cur = self.conn.execute(sql, params)
|
|
1454
|
-
self.
|
|
1564
|
+
if self._own_transaction():
|
|
1565
|
+
self.conn.commit()
|
|
1455
1566
|
return cur
|
|
1456
1567
|
|
|
1457
1568
|
def _fetchone(
|
|
@@ -2094,4 +2205,31 @@ CREATE TABLE IF NOT EXISTS lineage_event (
|
|
|
2094
2205
|
);
|
|
2095
2206
|
CREATE INDEX IF NOT EXISTS idx_lineage_entity ON lineage_event (entity_type, entity_id);
|
|
2096
2207
|
|
|
2208
|
+
-- ============================================================================
|
|
2209
|
+
-- Durable learning pipeline — cross-org job queue
|
|
2210
|
+
-- ============================================================================
|
|
2211
|
+
|
|
2212
|
+
CREATE TABLE IF NOT EXISTS learning_jobs (
|
|
2213
|
+
job_id TEXT PRIMARY KEY,
|
|
2214
|
+
org_id TEXT NOT NULL,
|
|
2215
|
+
user_id TEXT NOT NULL,
|
|
2216
|
+
job_type TEXT NOT NULL DEFAULT 'learning',
|
|
2217
|
+
latest_request_id TEXT,
|
|
2218
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
2219
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
2220
|
+
max_attempts INTEGER NOT NULL DEFAULT 3,
|
|
2221
|
+
claimed_by TEXT,
|
|
2222
|
+
claim_token TEXT,
|
|
2223
|
+
claim_expires_at TEXT,
|
|
2224
|
+
covers_through TEXT,
|
|
2225
|
+
force_extraction INTEGER NOT NULL DEFAULT 0,
|
|
2226
|
+
skip_aggregation INTEGER NOT NULL DEFAULT 0,
|
|
2227
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
|
2228
|
+
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2229
|
+
);
|
|
2230
|
+
CREATE UNIQUE INDEX IF NOT EXISTS learning_jobs_coalesce
|
|
2231
|
+
ON learning_jobs (org_id, user_id, job_type) WHERE status = 'pending';
|
|
2232
|
+
CREATE INDEX IF NOT EXISTS learning_jobs_poll
|
|
2233
|
+
ON learning_jobs (created_at) WHERE status IN ('pending','failed','claimed');
|
|
2234
|
+
|
|
2097
2235
|
"""
|
package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py
ADDED
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
"""SQLite implementation of the durable learning-job queue (Task 3)."""
|
|
2
|
+
|
|
3
|
+
import sqlite3
|
|
4
|
+
import time
|
|
5
|
+
import uuid
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from reflexio.server.services.storage.storage_base._learning_jobs import (
|
|
9
|
+
_ABSENCE_DONE_AFTER_SECONDS,
|
|
10
|
+
LearningJob,
|
|
11
|
+
LearningJobStoreABC,
|
|
12
|
+
LearningStatus,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
from ._base import SQLiteStorageBase, _epoch_to_iso, _iso_to_epoch
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _row_to_learning_job(row: sqlite3.Row) -> LearningJob:
|
|
19
|
+
"""Convert a sqlite3.Row from learning_jobs to a LearningJob dataclass."""
|
|
20
|
+
d = dict(row)
|
|
21
|
+
ct = d.get("covers_through")
|
|
22
|
+
return LearningJob(
|
|
23
|
+
job_id=d["job_id"],
|
|
24
|
+
org_id=d["org_id"],
|
|
25
|
+
user_id=d["user_id"],
|
|
26
|
+
job_type=d["job_type"],
|
|
27
|
+
latest_request_id=d.get("latest_request_id"),
|
|
28
|
+
status=d["status"],
|
|
29
|
+
attempts=d["attempts"],
|
|
30
|
+
claim_token=d.get("claim_token"),
|
|
31
|
+
covers_through=float(_iso_to_epoch(ct)) if ct else None,
|
|
32
|
+
force_extraction=bool(d.get("force_extraction", 0)),
|
|
33
|
+
skip_aggregation=bool(d.get("skip_aggregation", 0)),
|
|
34
|
+
max_attempts=int(d.get("max_attempts", 3)),
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class SQLiteLearningJobStoreMixin(LearningJobStoreABC):
|
|
39
|
+
"""SQLite implementation of the learning-job queue.
|
|
40
|
+
|
|
41
|
+
Relies on instance attributes provided by SQLiteStorageBase via MRO:
|
|
42
|
+
``conn``, ``_lock``, ``_own_transaction``, ``org_id``.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
# Type annotations for attributes/methods supplied by SQLiteStorageBase via MRO
|
|
46
|
+
_lock: Any
|
|
47
|
+
conn: sqlite3.Connection
|
|
48
|
+
org_id: str
|
|
49
|
+
_own_transaction: Any
|
|
50
|
+
_fetchall: Any
|
|
51
|
+
|
|
52
|
+
@SQLiteStorageBase.handle_exceptions
|
|
53
|
+
def enqueue_learning_job(
|
|
54
|
+
self,
|
|
55
|
+
*,
|
|
56
|
+
org_id: str,
|
|
57
|
+
user_id: str,
|
|
58
|
+
request_id: str,
|
|
59
|
+
covers_through: float,
|
|
60
|
+
job_type: str = "learning",
|
|
61
|
+
force_extraction: bool = False,
|
|
62
|
+
skip_aggregation: bool = False,
|
|
63
|
+
) -> str:
|
|
64
|
+
"""Coalescing upsert — safe to call inside a commit_scope."""
|
|
65
|
+
job_id = str(uuid.uuid4())
|
|
66
|
+
# int() truncates sub-second precision — intentional (second-precision epochs).
|
|
67
|
+
iso_covers = _epoch_to_iso(int(covers_through))
|
|
68
|
+
fe_int = int(force_extraction)
|
|
69
|
+
sa_int = int(skip_aggregation)
|
|
70
|
+
with self._lock:
|
|
71
|
+
own_txn = self._own_transaction()
|
|
72
|
+
try:
|
|
73
|
+
if own_txn:
|
|
74
|
+
self.conn.execute("BEGIN IMMEDIATE")
|
|
75
|
+
row = self.conn.execute(
|
|
76
|
+
"""
|
|
77
|
+
INSERT INTO learning_jobs
|
|
78
|
+
(job_id, org_id, user_id, job_type, latest_request_id,
|
|
79
|
+
covers_through, status, force_extraction, skip_aggregation,
|
|
80
|
+
created_at, updated_at)
|
|
81
|
+
VALUES
|
|
82
|
+
(?, ?, ?, ?, ?,
|
|
83
|
+
?, 'pending', ?, ?,
|
|
84
|
+
strftime('%Y-%m-%dT%H:%M:%fZ','now'),
|
|
85
|
+
strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
86
|
+
ON CONFLICT (org_id, user_id, job_type) WHERE status = 'pending'
|
|
87
|
+
DO UPDATE SET
|
|
88
|
+
latest_request_id = excluded.latest_request_id,
|
|
89
|
+
covers_through = CASE
|
|
90
|
+
WHEN learning_jobs.covers_through > excluded.covers_through
|
|
91
|
+
THEN learning_jobs.covers_through
|
|
92
|
+
ELSE excluded.covers_through
|
|
93
|
+
END,
|
|
94
|
+
force_extraction = excluded.force_extraction,
|
|
95
|
+
skip_aggregation = excluded.skip_aggregation,
|
|
96
|
+
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
|
|
97
|
+
RETURNING job_id
|
|
98
|
+
""",
|
|
99
|
+
(
|
|
100
|
+
job_id,
|
|
101
|
+
org_id,
|
|
102
|
+
user_id,
|
|
103
|
+
job_type,
|
|
104
|
+
request_id,
|
|
105
|
+
iso_covers,
|
|
106
|
+
fe_int,
|
|
107
|
+
sa_int,
|
|
108
|
+
),
|
|
109
|
+
).fetchone()
|
|
110
|
+
if own_txn:
|
|
111
|
+
self.conn.commit()
|
|
112
|
+
except Exception:
|
|
113
|
+
if own_txn:
|
|
114
|
+
self.conn.rollback()
|
|
115
|
+
raise
|
|
116
|
+
if row is None:
|
|
117
|
+
raise RuntimeError("enqueue_learning_job RETURNING job_id returned no row")
|
|
118
|
+
return str(row["job_id"])
|
|
119
|
+
|
|
120
|
+
@SQLiteStorageBase.handle_exceptions
|
|
121
|
+
def claim_learning_jobs(
|
|
122
|
+
self,
|
|
123
|
+
*,
|
|
124
|
+
claimed_by: str,
|
|
125
|
+
limit: int,
|
|
126
|
+
lease_seconds: int,
|
|
127
|
+
) -> list[LearningJob]:
|
|
128
|
+
"""BEGIN IMMEDIATE + SELECT + UPDATE to atomically claim jobs."""
|
|
129
|
+
with self._lock:
|
|
130
|
+
own_txn = self._own_transaction()
|
|
131
|
+
try:
|
|
132
|
+
if own_txn:
|
|
133
|
+
self.conn.execute("BEGIN IMMEDIATE")
|
|
134
|
+
|
|
135
|
+
# Find candidate job_ids using DB's now() to avoid clock skew.
|
|
136
|
+
# Include 'failed' so a failed-but-not-dead job is naturally
|
|
137
|
+
# reclaimable without a manual status reset.
|
|
138
|
+
candidate_rows = self.conn.execute(
|
|
139
|
+
"""
|
|
140
|
+
SELECT job_id FROM learning_jobs
|
|
141
|
+
WHERE org_id = ?
|
|
142
|
+
AND (
|
|
143
|
+
status = 'pending'
|
|
144
|
+
OR status = 'failed'
|
|
145
|
+
OR (status = 'claimed'
|
|
146
|
+
AND claim_expires_at < strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
147
|
+
)
|
|
148
|
+
ORDER BY created_at
|
|
149
|
+
LIMIT ?
|
|
150
|
+
""",
|
|
151
|
+
(self.org_id, limit),
|
|
152
|
+
).fetchall()
|
|
153
|
+
|
|
154
|
+
claimed: list[LearningJob] = []
|
|
155
|
+
for cand in candidate_rows:
|
|
156
|
+
job_id = cand["job_id"]
|
|
157
|
+
claim_token = str(uuid.uuid4())
|
|
158
|
+
updated = self.conn.execute(
|
|
159
|
+
"""
|
|
160
|
+
UPDATE learning_jobs SET
|
|
161
|
+
status = 'claimed',
|
|
162
|
+
claimed_by = ?,
|
|
163
|
+
claim_token = ?,
|
|
164
|
+
claim_expires_at = strftime(
|
|
165
|
+
'%Y-%m-%dT%H:%M:%fZ', 'now',
|
|
166
|
+
? || ' seconds'
|
|
167
|
+
),
|
|
168
|
+
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now'),
|
|
169
|
+
attempts = attempts + 1
|
|
170
|
+
WHERE job_id = ?
|
|
171
|
+
RETURNING *
|
|
172
|
+
""",
|
|
173
|
+
(claimed_by, claim_token, str(lease_seconds), job_id),
|
|
174
|
+
).fetchone()
|
|
175
|
+
if updated is not None:
|
|
176
|
+
claimed.append(_row_to_learning_job(updated))
|
|
177
|
+
|
|
178
|
+
if own_txn:
|
|
179
|
+
self.conn.commit()
|
|
180
|
+
except Exception:
|
|
181
|
+
if own_txn:
|
|
182
|
+
self.conn.rollback()
|
|
183
|
+
raise
|
|
184
|
+
|
|
185
|
+
return claimed
|
|
186
|
+
|
|
187
|
+
@SQLiteStorageBase.handle_exceptions
|
|
188
|
+
def heartbeat_learning_job(
|
|
189
|
+
self,
|
|
190
|
+
*,
|
|
191
|
+
job_id: str,
|
|
192
|
+
claim_token: str,
|
|
193
|
+
lease_seconds: int,
|
|
194
|
+
) -> bool:
|
|
195
|
+
"""Extend the lease; return True if the token is still live."""
|
|
196
|
+
with self._lock:
|
|
197
|
+
own_txn = self._own_transaction()
|
|
198
|
+
try:
|
|
199
|
+
if own_txn:
|
|
200
|
+
self.conn.execute("BEGIN IMMEDIATE")
|
|
201
|
+
cur = self.conn.execute(
|
|
202
|
+
"""
|
|
203
|
+
UPDATE learning_jobs SET
|
|
204
|
+
claim_expires_at = strftime(
|
|
205
|
+
'%Y-%m-%dT%H:%M:%fZ', 'now',
|
|
206
|
+
? || ' seconds'
|
|
207
|
+
),
|
|
208
|
+
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
|
|
209
|
+
WHERE job_id = ? AND claim_token = ? AND status = 'claimed'
|
|
210
|
+
""",
|
|
211
|
+
(str(lease_seconds), job_id, claim_token),
|
|
212
|
+
)
|
|
213
|
+
updated = cur.rowcount == 1
|
|
214
|
+
if own_txn:
|
|
215
|
+
self.conn.commit()
|
|
216
|
+
except Exception:
|
|
217
|
+
if own_txn:
|
|
218
|
+
self.conn.rollback()
|
|
219
|
+
raise
|
|
220
|
+
|
|
221
|
+
return updated
|
|
222
|
+
|
|
223
|
+
@SQLiteStorageBase.handle_exceptions
|
|
224
|
+
def complete_learning_job(
|
|
225
|
+
self,
|
|
226
|
+
*,
|
|
227
|
+
job_id: str,
|
|
228
|
+
claim_token: str,
|
|
229
|
+
) -> int:
|
|
230
|
+
"""Fenced completion — returns rowcount (0=superseded, 1=success).
|
|
231
|
+
|
|
232
|
+
Safe to call inside a commit_scope — no own BEGIN/COMMIT issued.
|
|
233
|
+
"""
|
|
234
|
+
with self._lock:
|
|
235
|
+
own_txn = self._own_transaction()
|
|
236
|
+
try:
|
|
237
|
+
if own_txn:
|
|
238
|
+
self.conn.execute("BEGIN IMMEDIATE")
|
|
239
|
+
cur = self.conn.execute(
|
|
240
|
+
"""
|
|
241
|
+
UPDATE learning_jobs SET
|
|
242
|
+
status = 'done',
|
|
243
|
+
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
|
|
244
|
+
WHERE job_id = ? AND claim_token = ? AND status = 'claimed'
|
|
245
|
+
""",
|
|
246
|
+
(job_id, claim_token),
|
|
247
|
+
)
|
|
248
|
+
rowcount = cur.rowcount
|
|
249
|
+
if own_txn:
|
|
250
|
+
self.conn.commit()
|
|
251
|
+
except Exception:
|
|
252
|
+
if own_txn:
|
|
253
|
+
self.conn.rollback()
|
|
254
|
+
raise
|
|
255
|
+
|
|
256
|
+
return rowcount
|
|
257
|
+
|
|
258
|
+
@SQLiteStorageBase.handle_exceptions
|
|
259
|
+
def fail_learning_job(
|
|
260
|
+
self,
|
|
261
|
+
*,
|
|
262
|
+
job_id: str,
|
|
263
|
+
claim_token: str,
|
|
264
|
+
dead: bool,
|
|
265
|
+
) -> None:
|
|
266
|
+
"""Fenced fail/dead transition — sets status, clears token for retry.
|
|
267
|
+
|
|
268
|
+
Does NOT increment attempts: claim_learning_jobs already incremented on
|
|
269
|
+
delivery. attempts tracks delivery count; fail only transitions status.
|
|
270
|
+
"""
|
|
271
|
+
new_status = "dead" if dead else "failed"
|
|
272
|
+
# Clear claim_token and claim_expires_at only for 'failed' so it's reclaimable.
|
|
273
|
+
# For 'dead', we keep claim_token set for auditability (won't be reclaimed anyway).
|
|
274
|
+
with self._lock:
|
|
275
|
+
own_txn = self._own_transaction()
|
|
276
|
+
try:
|
|
277
|
+
if own_txn:
|
|
278
|
+
self.conn.execute("BEGIN IMMEDIATE")
|
|
279
|
+
self.conn.execute(
|
|
280
|
+
"""
|
|
281
|
+
UPDATE learning_jobs SET
|
|
282
|
+
status = ?,
|
|
283
|
+
claim_token = CASE WHEN ? THEN claim_token ELSE NULL END,
|
|
284
|
+
claim_expires_at = CASE WHEN ? THEN claim_expires_at ELSE NULL END,
|
|
285
|
+
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
|
|
286
|
+
WHERE job_id = ? AND claim_token = ? AND status = 'claimed'
|
|
287
|
+
""",
|
|
288
|
+
(new_status, dead, dead, job_id, claim_token),
|
|
289
|
+
)
|
|
290
|
+
if own_txn:
|
|
291
|
+
self.conn.commit()
|
|
292
|
+
except Exception:
|
|
293
|
+
if own_txn:
|
|
294
|
+
self.conn.rollback()
|
|
295
|
+
raise
|
|
296
|
+
|
|
297
|
+
@SQLiteStorageBase.handle_exceptions
|
|
298
|
+
def list_org_ids_with_pending_learning_jobs(self) -> list[str]:
|
|
299
|
+
"""Distinct org_ids with actionable jobs (cross-org, not org-scoped).
|
|
300
|
+
|
|
301
|
+
Uses the DB's now() for the expired-lease comparison to avoid clock skew,
|
|
302
|
+
mirroring ``claim_learning_jobs``. Index-aided by ``learning_jobs_poll``
|
|
303
|
+
(partial index narrows the scan to non-terminal rows; org_id still requires
|
|
304
|
+
a heap fetch).
|
|
305
|
+
"""
|
|
306
|
+
rows = self._fetchall(
|
|
307
|
+
"""
|
|
308
|
+
SELECT DISTINCT org_id FROM learning_jobs
|
|
309
|
+
WHERE status = 'pending'
|
|
310
|
+
OR status = 'failed'
|
|
311
|
+
OR (status = 'claimed'
|
|
312
|
+
AND claim_expires_at < strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
313
|
+
ORDER BY org_id ASC
|
|
314
|
+
""",
|
|
315
|
+
(),
|
|
316
|
+
)
|
|
317
|
+
return [str(row["org_id"]) for row in rows]
|
|
318
|
+
|
|
319
|
+
@SQLiteStorageBase.handle_exceptions
|
|
320
|
+
def get_learning_status_for_request(
|
|
321
|
+
self,
|
|
322
|
+
*,
|
|
323
|
+
user_id: str,
|
|
324
|
+
request_created_at: float,
|
|
325
|
+
) -> LearningStatus:
|
|
326
|
+
"""Coverage-based status lookup (§3.6 rule).
|
|
327
|
+
|
|
328
|
+
Converts request_created_at epoch to ISO for lexicographic comparison
|
|
329
|
+
with stored covers_through ISO strings (same format, both UTC).
|
|
330
|
+
"""
|
|
331
|
+
req_iso = _epoch_to_iso(int(request_created_at))
|
|
332
|
+
rows = self._fetchall(
|
|
333
|
+
"SELECT status, covers_through FROM learning_jobs "
|
|
334
|
+
"WHERE org_id = ? AND user_id = ?",
|
|
335
|
+
(self.org_id, user_id),
|
|
336
|
+
)
|
|
337
|
+
|
|
338
|
+
has_pending = False
|
|
339
|
+
has_claimed_covering = False
|
|
340
|
+
has_dead_covering = False
|
|
341
|
+
has_failed = False
|
|
342
|
+
|
|
343
|
+
for row in rows:
|
|
344
|
+
status = row["status"]
|
|
345
|
+
ct: str | None = row["covers_through"]
|
|
346
|
+
covers = ct is not None and ct >= req_iso
|
|
347
|
+
|
|
348
|
+
if covers and status == "done":
|
|
349
|
+
return "done"
|
|
350
|
+
if covers and status == "claimed":
|
|
351
|
+
has_claimed_covering = True
|
|
352
|
+
if covers and status == "dead":
|
|
353
|
+
has_dead_covering = True
|
|
354
|
+
if status == "failed":
|
|
355
|
+
# 'failed' is reclaimable (attempts < max_attempts); treat as pending.
|
|
356
|
+
# Accumulate as a flag so a covering done row (encountered later in the
|
|
357
|
+
# iteration) is not shadowed by a failed row yielded earlier.
|
|
358
|
+
has_failed = True
|
|
359
|
+
if status == "pending":
|
|
360
|
+
has_pending = True
|
|
361
|
+
if status == "claimed":
|
|
362
|
+
# Deliberately treats any claimed job as covering, regardless of its
|
|
363
|
+
# covers_through value — it will extend the window once it completes.
|
|
364
|
+
has_claimed_covering = True
|
|
365
|
+
|
|
366
|
+
if has_claimed_covering:
|
|
367
|
+
return "processing"
|
|
368
|
+
if has_pending:
|
|
369
|
+
return "pending"
|
|
370
|
+
if has_failed:
|
|
371
|
+
return "pending"
|
|
372
|
+
if has_dead_covering:
|
|
373
|
+
return "failed"
|
|
374
|
+
# Absence semantics: terminal rows (done/dead) are GC'd after 24-72 h.
|
|
375
|
+
# Only treat absence as "done" once the request is old enough that a done
|
|
376
|
+
# row would have been reaped; a recent request with no rows is still pending.
|
|
377
|
+
if time.time() - request_created_at >= _ABSENCE_DONE_AFTER_SECONDS:
|
|
378
|
+
return "done"
|
|
379
|
+
return "pending"
|
|
@@ -127,6 +127,7 @@ class SQLiteLineageMixin:
|
|
|
127
127
|
conn: sqlite3.Connection
|
|
128
128
|
_lock: threading.RLock
|
|
129
129
|
org_id: str
|
|
130
|
+
_own_transaction: Any
|
|
130
131
|
|
|
131
132
|
def append_lineage_event(self, event: LineageEvent) -> int:
|
|
132
133
|
"""Append an event; idempotent on (org_id, entity_type, entity_id, op, request_id).
|
|
@@ -173,10 +174,12 @@ class SQLiteLineageMixin:
|
|
|
173
174
|
),
|
|
174
175
|
).fetchone()
|
|
175
176
|
eid = row[0] if row else None
|
|
176
|
-
self.
|
|
177
|
+
if self._own_transaction():
|
|
178
|
+
self.conn.commit()
|
|
177
179
|
return int(eid) if eid is not None else 0
|
|
178
180
|
last = cur.lastrowid
|
|
179
|
-
self.
|
|
181
|
+
if self._own_transaction():
|
|
182
|
+
self.conn.commit()
|
|
180
183
|
return int(last) if last is not None else 0
|
|
181
184
|
|
|
182
185
|
def get_lineage_events(
|
|
@@ -301,7 +304,8 @@ class SQLiteLineageMixin:
|
|
|
301
304
|
request_id=context.request_id,
|
|
302
305
|
reason=context.reason,
|
|
303
306
|
)
|
|
304
|
-
self.
|
|
307
|
+
if self._own_transaction():
|
|
308
|
+
self.conn.commit()
|
|
305
309
|
|
|
306
310
|
def supersede_record(
|
|
307
311
|
self,
|
|
@@ -343,7 +347,8 @@ class SQLiteLineageMixin:
|
|
|
343
347
|
(Status.SUPERSEDED.value, successor_id, _epoch_now(), incumbent_id),
|
|
344
348
|
)
|
|
345
349
|
if cur.rowcount == 0:
|
|
346
|
-
self.
|
|
350
|
+
if self._own_transaction():
|
|
351
|
+
self.conn.commit()
|
|
347
352
|
return False
|
|
348
353
|
_append_event_stmt(
|
|
349
354
|
self.conn,
|
|
@@ -357,7 +362,8 @@ class SQLiteLineageMixin:
|
|
|
357
362
|
request_id=context.request_id,
|
|
358
363
|
reason=context.reason,
|
|
359
364
|
)
|
|
360
|
-
self.
|
|
365
|
+
if self._own_transaction():
|
|
366
|
+
self.conn.commit()
|
|
361
367
|
return True
|
|
362
368
|
|
|
363
369
|
def purge_content(self, *, entity_type: EntityType, entity_id: str) -> bool:
|
|
@@ -32,6 +32,7 @@ class RequestMixin:
|
|
|
32
32
|
_fetchall: Any
|
|
33
33
|
_subject_ref_for_user_id: Any
|
|
34
34
|
_assert_subject_writable_locked: Any
|
|
35
|
+
_own_transaction: Any
|
|
35
36
|
|
|
36
37
|
# ------------------------------------------------------------------
|
|
37
38
|
# Request methods
|
|
@@ -42,8 +43,10 @@ class RequestMixin:
|
|
|
42
43
|
created_at_iso = _epoch_to_iso(request.created_at)
|
|
43
44
|
subject_ref = self._subject_ref_for_user_id(request.user_id)
|
|
44
45
|
with self._lock:
|
|
46
|
+
own_txn = self._own_transaction()
|
|
45
47
|
try:
|
|
46
|
-
|
|
48
|
+
if own_txn:
|
|
49
|
+
self.conn.execute("BEGIN IMMEDIATE")
|
|
47
50
|
self._assert_subject_writable_locked(subject_ref)
|
|
48
51
|
self.conn.execute(
|
|
49
52
|
"""INSERT OR REPLACE INTO requests
|
|
@@ -61,9 +64,11 @@ class RequestMixin:
|
|
|
61
64
|
subject_ref,
|
|
62
65
|
),
|
|
63
66
|
)
|
|
64
|
-
|
|
67
|
+
if own_txn:
|
|
68
|
+
self.conn.commit()
|
|
65
69
|
except Exception:
|
|
66
|
-
|
|
70
|
+
if own_txn:
|
|
71
|
+
self.conn.rollback()
|
|
67
72
|
raise
|
|
68
73
|
|
|
69
74
|
@SQLiteStorageBase.handle_exceptions
|