@memnexus-ai/cli 1.7.216 → 1.7.218
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/dist/commands/mcp.d.ts.map +1 -1
- package/dist/commands/mcp.js +16 -0
- package/dist/commands/mcp.js.map +1 -1
- package/dist/commands/memories.d.ts +173 -0
- package/dist/commands/memories.d.ts.map +1 -1
- package/dist/commands/memories.js +529 -1
- package/dist/commands/memories.js.map +1 -1
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -1
- package/dist/lib/commit-context/code-context.d.ts +31 -1
- package/dist/lib/commit-context/code-context.d.ts.map +1 -1
- package/dist/lib/commit-context/code-context.js +101 -2
- package/dist/lib/commit-context/code-context.js.map +1 -1
- package/package.json +2 -2
|
@@ -36,7 +36,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
36
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
exports.BULK_DELETE_LIST_PAGE_SIZE = exports.BULK_DELETE_MAX_RETRIES = exports.BULK_DELETE_PAGE_PAUSE_MS = exports.BULK_DELETE_MAX_IDS = exports.BULK_DELETE_GUARD_WINDOW_MS = exports.BULK_DELETE_GUARD_THRESHOLD = void 0;
|
|
39
|
+
exports.StdinTooLargeError = exports.BATCH_CREATE_ALLOWED_KEYS = exports.BATCH_CREATE_MAX_LINE_BYTES = exports.BATCH_CREATE_MAX_STDIN_BYTES = exports.BATCH_CREATE_MAX_RETRIES = exports.BATCH_CREATE_CONCURRENCY_MAX = exports.BATCH_CREATE_CONCURRENCY_MIN = exports.BATCH_CREATE_DEFAULT_CONCURRENCY = exports.BULK_DELETE_LIST_PAGE_SIZE = exports.BULK_DELETE_MAX_RETRIES = exports.BULK_DELETE_PAGE_PAUSE_MS = exports.BULK_DELETE_MAX_IDS = exports.BULK_DELETE_GUARD_WINDOW_MS = exports.BULK_DELETE_GUARD_THRESHOLD = void 0;
|
|
40
40
|
exports.getBulkDeleteGuardPath = getBulkDeleteGuardPath;
|
|
41
41
|
exports.readDeleteGuardTimestamps = readDeleteGuardTimestamps;
|
|
42
42
|
exports.recordDeletion = recordDeletion;
|
|
@@ -51,6 +51,16 @@ exports.sleep = sleep;
|
|
|
51
51
|
exports.bulkDeleteWithRetry = bulkDeleteWithRetry;
|
|
52
52
|
exports.runBulkDeletePaged = runBulkDeletePaged;
|
|
53
53
|
exports.collectAllMemoryIds = collectAllMemoryIds;
|
|
54
|
+
exports.isServiceUnavailableError = isServiceUnavailableError;
|
|
55
|
+
exports.isRetryableCreateError = isRetryableCreateError;
|
|
56
|
+
exports.isQuotaError = isQuotaError;
|
|
57
|
+
exports.parseJsonlCreateLine = parseJsonlCreateLine;
|
|
58
|
+
exports.resolveCreateConcurrency = resolveCreateConcurrency;
|
|
59
|
+
exports.batchCreateWithRetry = batchCreateWithRetry;
|
|
60
|
+
exports.runBatchCreate = runBatchCreate;
|
|
61
|
+
exports.buildBatchCreateItems = buildBatchCreateItems;
|
|
62
|
+
exports.readStdinRaw = readStdinRaw;
|
|
63
|
+
exports.handleStreamingCreate = handleStreamingCreate;
|
|
54
64
|
exports.registerMemoriesCommands = registerMemoriesCommands;
|
|
55
65
|
exports.registerMemoryRelationshipCommands = registerMemoryRelationshipCommands;
|
|
56
66
|
exports.registerMemoryTimelineCommand = registerMemoryTimelineCommand;
|
|
@@ -779,6 +789,506 @@ async function handleBatchDelete(rawIds, options, mode) {
|
|
|
779
789
|
throw error;
|
|
780
790
|
}
|
|
781
791
|
}
|
|
792
|
+
// ── Streaming batch create (PO CLI-latency directive, iteration 2) ────────────
|
|
793
|
+
// Scripted single-creates pay node's cold-start (0.3-0.5s) PLUS one HTTP round
|
|
794
|
+
// trip PER memory: 53 memories ≈ 1-2 min end-to-end. There is deliberately NO
|
|
795
|
+
// batch-create server endpoint yet (deferred — an L-effort item on the triage
|
|
796
|
+
// plan). This is a CLIENT-SIDE fix only: ONE process reads JSONL from stdin and
|
|
797
|
+
// fires N createMemory calls with connection reuse and bounded concurrency,
|
|
798
|
+
// targeting ~20-40s for the same 53 memories. Each item is still one HTTP
|
|
799
|
+
// request against the server; we cut the cold-start tax and serialize latency,
|
|
800
|
+
// not the request count.
|
|
801
|
+
/** Default parallel creates when MX_CREATE_CONCURRENCY is unset. */
|
|
802
|
+
exports.BATCH_CREATE_DEFAULT_CONCURRENCY = 4;
|
|
803
|
+
/** Clamp band for MX_CREATE_CONCURRENCY (mirrors the RECALL_MAX_MS clamp convention). */
|
|
804
|
+
exports.BATCH_CREATE_CONCURRENCY_MIN = 1;
|
|
805
|
+
exports.BATCH_CREATE_CONCURRENCY_MAX = 8;
|
|
806
|
+
/** Max retries per item on 429/503 before giving up (mirrors batch delete). */
|
|
807
|
+
exports.BATCH_CREATE_MAX_RETRIES = 3;
|
|
808
|
+
/**
|
|
809
|
+
* Hard cap on total stdin bytes read (128 MB) before we abort. A mistaken
|
|
810
|
+
* multi-GB pipe would otherwise accumulate unbounded and OOM the CLI
|
|
811
|
+
* (CWE-400, client-side); we fail closed with a clear error instead. Matches
|
|
812
|
+
* the 64KB-class file-bound precedent from v3.89a — generous enough that any
|
|
813
|
+
* legitimate JSONL batch fits, small enough that a runaway pipe stops early.
|
|
814
|
+
*/
|
|
815
|
+
exports.BATCH_CREATE_MAX_STDIN_BYTES = 128 * 1024 * 1024;
|
|
816
|
+
/**
|
|
817
|
+
* Hard cap on a single JSONL line length (1 MB). A line longer than this is
|
|
818
|
+
* rejected as a per-line parse error BEFORE JSON.parse (bounding parser memory),
|
|
819
|
+
* preserving the one-result-line-per-input-line contract instead of aborting
|
|
820
|
+
* the whole stream. A single memory body never legitimately approaches 1 MB.
|
|
821
|
+
*/
|
|
822
|
+
exports.BATCH_CREATE_MAX_LINE_BYTES = 1024 * 1024;
|
|
823
|
+
/**
|
|
824
|
+
* Fields a JSONL create line may carry. Mirrors the single-create option map
|
|
825
|
+
* (createMemory body). Any other key is rejected per-line — a typo like
|
|
826
|
+
* "contnet" fails that line loudly instead of silently dropping the content.
|
|
827
|
+
* `supersedes` is intentionally excluded: it is a separate relationship call in
|
|
828
|
+
* single-create, out of scope for the streaming path (noted in the PR).
|
|
829
|
+
*/
|
|
830
|
+
exports.BATCH_CREATE_ALLOWED_KEYS = new Set([
|
|
831
|
+
'content',
|
|
832
|
+
'name',
|
|
833
|
+
'topics',
|
|
834
|
+
'conversationId',
|
|
835
|
+
'memoryType',
|
|
836
|
+
'context',
|
|
837
|
+
'role',
|
|
838
|
+
'importance',
|
|
839
|
+
'codeContext',
|
|
840
|
+
'taskId',
|
|
841
|
+
'outcome',
|
|
842
|
+
'distill',
|
|
843
|
+
]);
|
|
844
|
+
/** True when an error is an HTTP 503 (service unavailable — transient). */
|
|
845
|
+
function isServiceUnavailableError(error) {
|
|
846
|
+
const status = error?.status ?? error?.statusCode ?? error?.response?.status;
|
|
847
|
+
return status === 503;
|
|
848
|
+
}
|
|
849
|
+
/** True when a create should be retried: 429 (rate limited) or 503 (transient). */
|
|
850
|
+
function isRetryableCreateError(error) {
|
|
851
|
+
return isRateLimitError(error) || isServiceUnavailableError(error);
|
|
852
|
+
}
|
|
853
|
+
/** True when an error is an HTTP 402 (quota / payment required). */
|
|
854
|
+
function isQuotaError(error) {
|
|
855
|
+
const status = error?.status ?? error?.statusCode ?? error?.response?.status;
|
|
856
|
+
return status === 402;
|
|
857
|
+
}
|
|
858
|
+
/**
|
|
859
|
+
* Parse and validate one JSONL line into a createMemory body.
|
|
860
|
+
*
|
|
861
|
+
* Returns `{ data }` on success or `{ error }` on failure. Rejects: non-JSON,
|
|
862
|
+
* non-object lines, unknown keys, and a missing/blank `content`. `conversationId`
|
|
863
|
+
* defaults to "NEW" (mirrors single-create). `topics` accepts a JSON array or a
|
|
864
|
+
* comma-separated string; `importance` accepts a number or numeric string.
|
|
865
|
+
*/
|
|
866
|
+
function parseJsonlCreateLine(raw) {
|
|
867
|
+
// Bound per-line memory BEFORE JSON.parse: reject an oversized line as a
|
|
868
|
+
// per-line error so the stream keeps going (one result line per input line).
|
|
869
|
+
const lineBytes = Buffer.byteLength(raw, 'utf8');
|
|
870
|
+
if (lineBytes > exports.BATCH_CREATE_MAX_LINE_BYTES) {
|
|
871
|
+
return {
|
|
872
|
+
error: `line too large (${lineBytes} bytes > ${exports.BATCH_CREATE_MAX_LINE_BYTES} byte limit)`,
|
|
873
|
+
};
|
|
874
|
+
}
|
|
875
|
+
let parsed;
|
|
876
|
+
try {
|
|
877
|
+
parsed = JSON.parse(raw);
|
|
878
|
+
}
|
|
879
|
+
catch {
|
|
880
|
+
return { error: 'invalid JSON' };
|
|
881
|
+
}
|
|
882
|
+
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
883
|
+
return { error: 'line must be a JSON object' };
|
|
884
|
+
}
|
|
885
|
+
const obj = parsed;
|
|
886
|
+
const unknown = Object.keys(obj).filter((k) => !exports.BATCH_CREATE_ALLOWED_KEYS.has(k));
|
|
887
|
+
if (unknown.length > 0) {
|
|
888
|
+
return { error: `unknown field(s): ${unknown.join(', ')}` };
|
|
889
|
+
}
|
|
890
|
+
if (typeof obj.content !== 'string' || obj.content.trim().length === 0) {
|
|
891
|
+
return { error: 'content is required and must be a non-empty string' };
|
|
892
|
+
}
|
|
893
|
+
if ('codeContext' in obj && obj.codeContext !== undefined) {
|
|
894
|
+
const cc = obj.codeContext;
|
|
895
|
+
if (cc === null || typeof cc !== 'object' || Array.isArray(cc)) {
|
|
896
|
+
return { error: 'codeContext must be a JSON object' };
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
let topics;
|
|
900
|
+
if ('topics' in obj && obj.topics !== undefined) {
|
|
901
|
+
if (Array.isArray(obj.topics)) {
|
|
902
|
+
topics = obj.topics.map((t) => String(t));
|
|
903
|
+
}
|
|
904
|
+
else if (typeof obj.topics === 'string') {
|
|
905
|
+
topics = obj.topics
|
|
906
|
+
.split(',')
|
|
907
|
+
.map((t) => t.trim())
|
|
908
|
+
.filter((t) => t.length > 0);
|
|
909
|
+
}
|
|
910
|
+
else {
|
|
911
|
+
return { error: 'topics must be an array or comma-separated string' };
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
let importance;
|
|
915
|
+
if ('importance' in obj && obj.importance !== undefined) {
|
|
916
|
+
const n = typeof obj.importance === 'string' ? parseFloat(obj.importance) : obj.importance;
|
|
917
|
+
if (typeof n !== 'number' || !Number.isFinite(n)) {
|
|
918
|
+
return { error: 'importance must be a number between 0 and 1' };
|
|
919
|
+
}
|
|
920
|
+
importance = n;
|
|
921
|
+
}
|
|
922
|
+
const conversationId = typeof obj.conversationId === 'string' && obj.conversationId.trim().length > 0
|
|
923
|
+
? obj.conversationId
|
|
924
|
+
: 'NEW';
|
|
925
|
+
const data = {
|
|
926
|
+
conversationId,
|
|
927
|
+
content: obj.content,
|
|
928
|
+
};
|
|
929
|
+
if (obj.name !== undefined)
|
|
930
|
+
data.name = obj.name;
|
|
931
|
+
if (topics !== undefined)
|
|
932
|
+
data.topics = topics;
|
|
933
|
+
// Defaults parity with single-create (memories.ts create action): role
|
|
934
|
+
// defaults to 'user'; memoryType defaults by name-presence (named memories are
|
|
935
|
+
// 'procedural', otherwise 'episodic'). Applying them here means batch and
|
|
936
|
+
// single create produce identical memories for identical input fields.
|
|
937
|
+
data.role = obj.role !== undefined ? obj.role : 'user';
|
|
938
|
+
data.memoryType =
|
|
939
|
+
obj.memoryType !== undefined
|
|
940
|
+
? obj.memoryType
|
|
941
|
+
: obj.name !== undefined
|
|
942
|
+
? 'procedural'
|
|
943
|
+
: 'episodic';
|
|
944
|
+
if (obj.context !== undefined)
|
|
945
|
+
data.context = obj.context;
|
|
946
|
+
if (importance !== undefined)
|
|
947
|
+
data.importance = importance;
|
|
948
|
+
if (obj.codeContext !== undefined)
|
|
949
|
+
data.codeContext = obj.codeContext;
|
|
950
|
+
if (obj.taskId !== undefined)
|
|
951
|
+
data.taskId = obj.taskId;
|
|
952
|
+
if (obj.outcome !== undefined)
|
|
953
|
+
data.outcome = obj.outcome;
|
|
954
|
+
if (obj.distill !== undefined)
|
|
955
|
+
data.distill = obj.distill;
|
|
956
|
+
return { data };
|
|
957
|
+
}
|
|
958
|
+
/**
|
|
959
|
+
* Resolve the create concurrency from MX_CREATE_CONCURRENCY, clamped to
|
|
960
|
+
* [MIN, MAX]. Out-of-band values are clamped to the nearest bound (non-numeric →
|
|
961
|
+
* default) and warn once on stderr — mirrors the RECALL_MAX_MS clamp convention
|
|
962
|
+
* (import-time band + single WARN).
|
|
963
|
+
*/
|
|
964
|
+
function resolveCreateConcurrency(raw, warn = (m) => console.warn(chalk_1.default.yellow(m))) {
|
|
965
|
+
if (raw === undefined || raw === '')
|
|
966
|
+
return exports.BATCH_CREATE_DEFAULT_CONCURRENCY;
|
|
967
|
+
const n = parseInt(raw, 10);
|
|
968
|
+
if (!Number.isFinite(n)) {
|
|
969
|
+
warn(`MX_CREATE_CONCURRENCY=${raw} is not a number, using default ${exports.BATCH_CREATE_DEFAULT_CONCURRENCY}`);
|
|
970
|
+
return exports.BATCH_CREATE_DEFAULT_CONCURRENCY;
|
|
971
|
+
}
|
|
972
|
+
if (n < exports.BATCH_CREATE_CONCURRENCY_MIN || n > exports.BATCH_CREATE_CONCURRENCY_MAX) {
|
|
973
|
+
const clamped = Math.max(exports.BATCH_CREATE_CONCURRENCY_MIN, Math.min(exports.BATCH_CREATE_CONCURRENCY_MAX, n));
|
|
974
|
+
warn(`MX_CREATE_CONCURRENCY=${raw} out of range [${exports.BATCH_CREATE_CONCURRENCY_MIN},${exports.BATCH_CREATE_CONCURRENCY_MAX}], clamped to ${clamped}`);
|
|
975
|
+
return clamped;
|
|
976
|
+
}
|
|
977
|
+
return n;
|
|
978
|
+
}
|
|
979
|
+
/**
|
|
980
|
+
* Create one memory, retrying on 429/503 up to `maxRetries` times, honoring the
|
|
981
|
+
* server's Retry-After hint (falls back to linear backoff). A 402 (quota) is
|
|
982
|
+
* NOT retried — it rethrows immediately so the pool can stop the stream early.
|
|
983
|
+
*/
|
|
984
|
+
async function batchCreateWithRetry(data, deps) {
|
|
985
|
+
const maxRetries = deps.maxRetries ?? exports.BATCH_CREATE_MAX_RETRIES;
|
|
986
|
+
const defaultRetryMs = deps.defaultRetryMs ?? 1000;
|
|
987
|
+
let attempt = 0;
|
|
988
|
+
for (;;) {
|
|
989
|
+
try {
|
|
990
|
+
return await deps.createFn(data);
|
|
991
|
+
}
|
|
992
|
+
catch (error) {
|
|
993
|
+
if (isQuotaError(error))
|
|
994
|
+
throw error;
|
|
995
|
+
if (isRetryableCreateError(error) && attempt < maxRetries) {
|
|
996
|
+
const waitMs = getRetryAfterMs(error) ?? defaultRetryMs * (attempt + 1);
|
|
997
|
+
deps.onRateLimited?.(waitMs);
|
|
998
|
+
await deps.sleep(waitMs);
|
|
999
|
+
attempt++;
|
|
1000
|
+
continue;
|
|
1001
|
+
}
|
|
1002
|
+
throw error;
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
/**
|
|
1007
|
+
* Run `items` through a bounded-concurrency worker pool. Each valid line becomes
|
|
1008
|
+
* one createMemory call with per-item 429/503 retry; each parse-error line
|
|
1009
|
+
* resolves immediately as a failure. Results are emitted via `onResult` in
|
|
1010
|
+
* COMPLETION order, with the input line number attached.
|
|
1011
|
+
*
|
|
1012
|
+
* A 402 (quota exhausted) STOPS the stream early: the offending line is reported
|
|
1013
|
+
* as failed, no new lines are pulled, in-flight creates drain, and every
|
|
1014
|
+
* not-yet-pulled line is emitted as a not-attempted failure in line order. This
|
|
1015
|
+
* mirrors the batch-delete early-abort contract — the caller always gets one
|
|
1016
|
+
* result line per input line, never a dropped report (the v3.90 lesson).
|
|
1017
|
+
*
|
|
1018
|
+
* If the sink (`onResult`, i.e. stdout) THROWS — the classic `... | head`
|
|
1019
|
+
* EPIPE — that is a distinct terminal condition, NOT a create failure. The
|
|
1020
|
+
* create for that line already succeeded and is counted; we stop pulling new
|
|
1021
|
+
* work, emit nothing further to the dead sink, and set `outputClosed` so the
|
|
1022
|
+
* caller can print an accounting to stderr (which is still open). `emit` is
|
|
1023
|
+
* guarded so a sink throw can never be misread as a create failure (which would
|
|
1024
|
+
* duplicate the line) nor propagate uncaught.
|
|
1025
|
+
*/
|
|
1026
|
+
async function runBatchCreate(items, deps) {
|
|
1027
|
+
const concurrency = Math.max(1, deps.concurrency);
|
|
1028
|
+
const summary = {
|
|
1029
|
+
total: items.length,
|
|
1030
|
+
ok: 0,
|
|
1031
|
+
failed: 0,
|
|
1032
|
+
notAttempted: 0,
|
|
1033
|
+
quotaStopped: false,
|
|
1034
|
+
rateLimitedCount: 0,
|
|
1035
|
+
delivered: 0,
|
|
1036
|
+
outputClosed: false,
|
|
1037
|
+
};
|
|
1038
|
+
const attempted = new Array(items.length).fill(false);
|
|
1039
|
+
let next = 0;
|
|
1040
|
+
let stopped = false;
|
|
1041
|
+
let sinkClosed = false;
|
|
1042
|
+
// Guarded sink write: never throws. A sink failure (EPIPE) becomes a terminal
|
|
1043
|
+
// stop condition instead of an exception that the create catch would misread.
|
|
1044
|
+
const emit = (r) => {
|
|
1045
|
+
if (sinkClosed)
|
|
1046
|
+
return;
|
|
1047
|
+
try {
|
|
1048
|
+
deps.onResult?.(r);
|
|
1049
|
+
summary.delivered++;
|
|
1050
|
+
}
|
|
1051
|
+
catch {
|
|
1052
|
+
sinkClosed = true;
|
|
1053
|
+
stopped = true;
|
|
1054
|
+
summary.outputClosed = true;
|
|
1055
|
+
}
|
|
1056
|
+
};
|
|
1057
|
+
async function worker() {
|
|
1058
|
+
for (;;) {
|
|
1059
|
+
if (stopped)
|
|
1060
|
+
return;
|
|
1061
|
+
const i = next++;
|
|
1062
|
+
if (i >= items.length)
|
|
1063
|
+
return;
|
|
1064
|
+
attempted[i] = true;
|
|
1065
|
+
const item = items[i];
|
|
1066
|
+
// Compute the result BEFORE emitting. The success emit lives OUTSIDE the
|
|
1067
|
+
// create try/catch so a sink throw on a successful line is handled by the
|
|
1068
|
+
// guarded emit — never caught here as a bogus create failure.
|
|
1069
|
+
let result;
|
|
1070
|
+
if (item.parseError) {
|
|
1071
|
+
summary.failed++;
|
|
1072
|
+
result = { line: item.line, ok: false, error: item.parseError, code: 'PARSE_ERROR' };
|
|
1073
|
+
}
|
|
1074
|
+
else {
|
|
1075
|
+
try {
|
|
1076
|
+
const res = await batchCreateWithRetry(item.data, {
|
|
1077
|
+
createFn: deps.createFn,
|
|
1078
|
+
sleep: deps.sleep,
|
|
1079
|
+
maxRetries: deps.maxRetries,
|
|
1080
|
+
defaultRetryMs: deps.defaultRetryMs,
|
|
1081
|
+
onRateLimited: () => {
|
|
1082
|
+
summary.rateLimitedCount++;
|
|
1083
|
+
},
|
|
1084
|
+
});
|
|
1085
|
+
summary.ok++;
|
|
1086
|
+
result = { line: item.line, ok: true, id: res?.id };
|
|
1087
|
+
}
|
|
1088
|
+
catch (error) {
|
|
1089
|
+
summary.failed++;
|
|
1090
|
+
if (isQuotaError(error)) {
|
|
1091
|
+
// Quota exhausted — pointless to keep hammering. Stop pulling new
|
|
1092
|
+
// lines; in-flight workers finish, remaining lines are not-attempted.
|
|
1093
|
+
stopped = true;
|
|
1094
|
+
summary.quotaStopped = true;
|
|
1095
|
+
emit({ line: item.line, ok: false, error: errorMessage(error), code: 'QUOTA_STOPPED' });
|
|
1096
|
+
return;
|
|
1097
|
+
}
|
|
1098
|
+
result = {
|
|
1099
|
+
line: item.line,
|
|
1100
|
+
ok: false,
|
|
1101
|
+
error: errorMessage(error),
|
|
1102
|
+
code: 'CREATE_FAILED',
|
|
1103
|
+
};
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
emit(result);
|
|
1107
|
+
// If the sink died on that write, stop cleanly (no further stdout writes).
|
|
1108
|
+
if (stopped)
|
|
1109
|
+
return;
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
const workerCount = Math.min(concurrency, items.length);
|
|
1113
|
+
await Promise.all(Array.from({ length: workerCount }, () => worker()));
|
|
1114
|
+
// Lines the pool never pulled (only possible after an early stop — quota or
|
|
1115
|
+
// sink close) are reported as not-attempted in line order — one result line
|
|
1116
|
+
// per input line. Under a sink close these emits are no-ops (sink is dead) but
|
|
1117
|
+
// are still COUNTED so the stderr accounting is complete.
|
|
1118
|
+
if (stopped) {
|
|
1119
|
+
const closed = summary.outputClosed;
|
|
1120
|
+
const code = closed ? 'OUTPUT_CLOSED' : 'NOT_ATTEMPTED';
|
|
1121
|
+
const reason = closed
|
|
1122
|
+
? 'not attempted (output pipe closed early)'
|
|
1123
|
+
: 'not attempted (stream stopped after quota exhaustion)';
|
|
1124
|
+
for (let i = 0; i < items.length; i++) {
|
|
1125
|
+
if (!attempted[i]) {
|
|
1126
|
+
summary.notAttempted++;
|
|
1127
|
+
emit({ line: items[i].line, ok: false, error: reason, code });
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
return summary;
|
|
1132
|
+
}
|
|
1133
|
+
/**
|
|
1134
|
+
* Split raw stdin into non-blank lines, each paired with its 1-based input line
|
|
1135
|
+
* number and parse result. Blank/whitespace-only lines are skipped silently (a
|
|
1136
|
+
* trailing newline is normal in JSONL and is not an error).
|
|
1137
|
+
*/
|
|
1138
|
+
function buildBatchCreateItems(raw) {
|
|
1139
|
+
const items = [];
|
|
1140
|
+
const lines = raw.split(/\r?\n/);
|
|
1141
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1142
|
+
const line = lines[i];
|
|
1143
|
+
if (line.trim().length === 0)
|
|
1144
|
+
continue;
|
|
1145
|
+
const parsed = parseJsonlCreateLine(line);
|
|
1146
|
+
items.push({ line: i + 1, data: parsed.data, parseError: parsed.error });
|
|
1147
|
+
}
|
|
1148
|
+
return items;
|
|
1149
|
+
}
|
|
1150
|
+
/** Thrown by readStdinRaw when accumulated stdin exceeds the byte cap. */
|
|
1151
|
+
class StdinTooLargeError extends Error {
|
|
1152
|
+
bytes;
|
|
1153
|
+
limit;
|
|
1154
|
+
code = 'STDIN_TOO_LARGE';
|
|
1155
|
+
constructor(bytes, limit) {
|
|
1156
|
+
super(`stdin exceeded the ${limit}-byte limit (read > ${bytes} bytes)`);
|
|
1157
|
+
this.bytes = bytes;
|
|
1158
|
+
this.limit = limit;
|
|
1159
|
+
this.name = 'StdinTooLargeError';
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1162
|
+
exports.StdinTooLargeError = StdinTooLargeError;
|
|
1163
|
+
/**
|
|
1164
|
+
* Read all of stdin as a single string (no whitespace splitting), aborting once
|
|
1165
|
+
* accumulated bytes exceed `maxBytes`. Without the cap a mistaken multi-GB pipe
|
|
1166
|
+
* would grow `data` unbounded and OOM the CLI (CWE-400); we fail closed with a
|
|
1167
|
+
* StdinTooLargeError instead. `maxBytes` is injectable for tests; production
|
|
1168
|
+
* uses BATCH_CREATE_MAX_STDIN_BYTES.
|
|
1169
|
+
*/
|
|
1170
|
+
async function readStdinRaw(maxBytes = exports.BATCH_CREATE_MAX_STDIN_BYTES) {
|
|
1171
|
+
if (process.stdin.isTTY)
|
|
1172
|
+
return '';
|
|
1173
|
+
return new Promise((resolve, reject) => {
|
|
1174
|
+
let data = '';
|
|
1175
|
+
let bytes = 0;
|
|
1176
|
+
let aborted = false;
|
|
1177
|
+
process.stdin.setEncoding('utf8');
|
|
1178
|
+
process.stdin.on('readable', () => {
|
|
1179
|
+
if (aborted)
|
|
1180
|
+
return;
|
|
1181
|
+
let chunk;
|
|
1182
|
+
while ((chunk = process.stdin.read()) !== null) {
|
|
1183
|
+
bytes += Buffer.byteLength(chunk, 'utf8');
|
|
1184
|
+
if (bytes > maxBytes) {
|
|
1185
|
+
aborted = true;
|
|
1186
|
+
// Stop draining and tear down the stream rather than keep buffering.
|
|
1187
|
+
if (typeof process.stdin.destroy === 'function') {
|
|
1188
|
+
process.stdin.destroy();
|
|
1189
|
+
}
|
|
1190
|
+
reject(new StdinTooLargeError(bytes, maxBytes));
|
|
1191
|
+
return;
|
|
1192
|
+
}
|
|
1193
|
+
data += chunk;
|
|
1194
|
+
}
|
|
1195
|
+
});
|
|
1196
|
+
process.stdin.on('end', () => {
|
|
1197
|
+
if (!aborted)
|
|
1198
|
+
resolve(data);
|
|
1199
|
+
});
|
|
1200
|
+
process.stdin.on('error', () => {
|
|
1201
|
+
if (!aborted)
|
|
1202
|
+
resolve(data);
|
|
1203
|
+
});
|
|
1204
|
+
});
|
|
1205
|
+
}
|
|
1206
|
+
/** Issue one createMemory via the SDK and normalize the id out of the response. */
|
|
1207
|
+
async function createOneViaSdk(data) {
|
|
1208
|
+
// The global fetch wrapper in index.ts attaches X-MX-Client AND the v3.89a
|
|
1209
|
+
// per-project attribution headers, so batch-created memories carry attribution
|
|
1210
|
+
// automatically — no per-item work here. codeContext supplied on a line still
|
|
1211
|
+
// rides in the request body untouched.
|
|
1212
|
+
const response = await (0, api_client_1.getSdk)().memories.createMemory(data);
|
|
1213
|
+
const memory = response?.data?.data ?? response?.data;
|
|
1214
|
+
return { id: memory?.id };
|
|
1215
|
+
}
|
|
1216
|
+
/**
|
|
1217
|
+
* `mx memories create --stdin-jsonl`: streaming batch create. Reads JSONL from
|
|
1218
|
+
* stdin (one createMemory body per line), fires bounded-concurrency creates, and
|
|
1219
|
+
* writes one JSONL result line per input line to stdout in completion order.
|
|
1220
|
+
*
|
|
1221
|
+
* stdout is PURE JSONL: the ONLY writes are the per-line result objects below.
|
|
1222
|
+
* Every status/warning/summary goes to stderr. There is no session-preamble
|
|
1223
|
+
* printer on this command path, so the #4176-class stdout-pollution bug cannot
|
|
1224
|
+
* occur here — but the split is enforced deliberately so a future preamble can
|
|
1225
|
+
* never leak into the machine-readable stream.
|
|
1226
|
+
*/
|
|
1227
|
+
async function handleStreamingCreate(maxStdinBytes = exports.BATCH_CREATE_MAX_STDIN_BYTES) {
|
|
1228
|
+
const concurrency = resolveCreateConcurrency(process.env.MX_CREATE_CONCURRENCY);
|
|
1229
|
+
let raw;
|
|
1230
|
+
try {
|
|
1231
|
+
raw = await readStdinRaw(maxStdinBytes);
|
|
1232
|
+
}
|
|
1233
|
+
catch (error) {
|
|
1234
|
+
if (error?.code === 'STDIN_TOO_LARGE') {
|
|
1235
|
+
// Fail closed: nothing was attempted, so nothing is emitted on stdout.
|
|
1236
|
+
console.error(chalk_1.default.red(`Error: stdin exceeded the ${maxStdinBytes}-byte limit; aborted before creating anything. ` +
|
|
1237
|
+
'Split the input into smaller batches.'));
|
|
1238
|
+
process.exitCode = 1;
|
|
1239
|
+
return;
|
|
1240
|
+
}
|
|
1241
|
+
throw error;
|
|
1242
|
+
}
|
|
1243
|
+
const items = buildBatchCreateItems(raw);
|
|
1244
|
+
if (items.length === 0) {
|
|
1245
|
+
console.error(chalk_1.default.red('Error: --stdin-jsonl expects one JSON object per line on stdin; got none.'));
|
|
1246
|
+
process.exitCode = 1;
|
|
1247
|
+
return;
|
|
1248
|
+
}
|
|
1249
|
+
// Auto-attach the Claude Code session id to every line, exactly as single
|
|
1250
|
+
// create does (create action: readSessionContext() when in AI-agent context),
|
|
1251
|
+
// so batch-created memories group into the same conversation/session.
|
|
1252
|
+
const claudeSession = config_1.config.isAIAgentContext() ? (0, session_reader_1.readSessionContext)() : undefined;
|
|
1253
|
+
const claudeSessionId = claudeSession?.sessionId || undefined;
|
|
1254
|
+
const summary = await runBatchCreate(items, {
|
|
1255
|
+
createFn: (data) => createOneViaSdk(claudeSessionId ? { ...data, claudeSessionId } : data),
|
|
1256
|
+
sleep,
|
|
1257
|
+
concurrency,
|
|
1258
|
+
onResult: (r) => {
|
|
1259
|
+
// Pure JSONL on stdout — the ONLY stdout write on this path. A broken pipe
|
|
1260
|
+
// (reader closed early) throws EPIPE here; runBatchCreate's guarded emit
|
|
1261
|
+
// catches it and stops cleanly (summary.outputClosed), handled below.
|
|
1262
|
+
process.stdout.write(JSON.stringify(r) + '\n');
|
|
1263
|
+
},
|
|
1264
|
+
});
|
|
1265
|
+
// Terminal: the reader closed stdout early (EPIPE). stdout is dead but stderr
|
|
1266
|
+
// is still open — write a distinct accounting so the caller knows exactly how
|
|
1267
|
+
// many results were delivered vs. created vs. never attempted. Creates are not
|
|
1268
|
+
// idempotent, so this reconciliation matters.
|
|
1269
|
+
if (summary.outputClosed) {
|
|
1270
|
+
console.error(chalk_1.default.red(`Error: output pipe closed early — ${summary.delivered} result line(s) delivered, ` +
|
|
1271
|
+
`${summary.ok} created, ${summary.notAttempted} not attempted. ` +
|
|
1272
|
+
'Reconcile against the delivered result lines before re-running (creates are not idempotent).'));
|
|
1273
|
+
process.exitCode = 1;
|
|
1274
|
+
return;
|
|
1275
|
+
}
|
|
1276
|
+
// Summary + any early-stop notice go to stderr so stdout stays pure JSONL.
|
|
1277
|
+
if (summary.quotaStopped) {
|
|
1278
|
+
console.error(chalk_1.default.red(`Stopped early on quota (402): ${summary.ok} created, ${summary.failed} failed, ${summary.notAttempted} not attempted.`));
|
|
1279
|
+
}
|
|
1280
|
+
else {
|
|
1281
|
+
const rl = summary.rateLimitedCount > 0
|
|
1282
|
+
? `, ${summary.rateLimitedCount} rate-limited retr${summary.rateLimitedCount === 1 ? 'y' : 'ies'}`
|
|
1283
|
+
: '';
|
|
1284
|
+
console.error(`Batch create: ${summary.ok} created, ${summary.failed} failed of ${summary.total}${rl} (concurrency ${concurrency}).`);
|
|
1285
|
+
}
|
|
1286
|
+
// Non-zero exit when anything failed or was not attempted, so scripts detect
|
|
1287
|
+
// partial failure — partial results are still fully emitted above.
|
|
1288
|
+
if (summary.failed > 0 || summary.notAttempted > 0) {
|
|
1289
|
+
process.exitCode = 1;
|
|
1290
|
+
}
|
|
1291
|
+
}
|
|
782
1292
|
/**
|
|
783
1293
|
* Register memory management commands
|
|
784
1294
|
* @param program - Commander program instance
|
|
@@ -1132,9 +1642,27 @@ function registerMemoriesCommands(program) {
|
|
|
1132
1642
|
.option('--outcome <outcome>', 'Task outcome: resolved | unresolved')
|
|
1133
1643
|
.option('--distill', 'Request LLM distillation into structured 5-section notebook (Pro/Enterprise only)')
|
|
1134
1644
|
.option('--interactive', 'Interactive mode')
|
|
1645
|
+
.option('--stdin-jsonl', 'Streaming batch create: read JSONL from stdin (one createMemory object per line), ' +
|
|
1646
|
+
'create with bounded concurrency (MX_CREATE_CONCURRENCY, default 4), and emit one ' +
|
|
1647
|
+
'JSONL result line per input line on stdout ({line, ok, id?, error?, code?}). On ' +
|
|
1648
|
+
'failure, code is one of PARSE_ERROR | CREATE_FAILED | QUOTA_STOPPED | NOT_ATTEMPTED | ' +
|
|
1649
|
+
'OUTPUT_CLOSED. Exits non-zero if any line failed.')
|
|
1135
1650
|
.option('--format <format>', 'Output format (json|table|yaml)')
|
|
1136
1651
|
.action(async (options) => {
|
|
1137
1652
|
try {
|
|
1653
|
+
// Streaming batch create path. Mutually exclusive with the single-create
|
|
1654
|
+
// content/interactive flags — the input is stdin JSONL, not CLI flags.
|
|
1655
|
+
if (options.stdinJsonl) {
|
|
1656
|
+
const conflicting = ['content', 'interactive', 'supersedes'].filter((k) => options[k]);
|
|
1657
|
+
if (conflicting.length > 0) {
|
|
1658
|
+
console.error(chalk_1.default.red(`Error: --stdin-jsonl cannot be combined with --${conflicting.join(', --')}. ` +
|
|
1659
|
+
'Pass all fields as JSON per line on stdin instead.'));
|
|
1660
|
+
process.exitCode = 1;
|
|
1661
|
+
return;
|
|
1662
|
+
}
|
|
1663
|
+
await handleStreamingCreate();
|
|
1664
|
+
return;
|
|
1665
|
+
}
|
|
1138
1666
|
// Build codeContext from --code-context JSON and/or convenience flags
|
|
1139
1667
|
let codeContext;
|
|
1140
1668
|
if (options.codeContext) {
|