maka-agent 0.2.2-dev.101.20261026 → 0.2.2-dev.105.20261029
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.txt +2 -2
- package/native/runtime-host-peer/prebuilds/win32-x64/maka_runtime_host_peer.node +0 -0
- package/native/runtime-host-windows-task-launcher/prebuilds/win32-x64/maka-runtime-host-task-launcher.exe +0 -0
- package/node_modules/@maka/core/dist/runtime-policy/connection-catalog-codec.js +6 -0
- package/node_modules/@maka/runtime/dist/ai-sdk-turn.js +21 -7
- package/node_modules/@maka/runtime/dist/history-compact-checkpoint.js +32 -4
- package/node_modules/@maka/runtime/dist/model-fetcher.js +7 -0
- package/node_modules/@maka/runtime/dist/openai-codex-history-compactor.js +13 -4
- package/node_modules/@maka/runtime-host/dist/assistant/index-initiative/assistant.js +3 -0
- package/node_modules/@maka/runtime-host/dist/assistant/index-initiative/controller.js +27 -2
- package/node_modules/@maka/runtime-host/dist/assistant/index-initiative/notebook.js +73 -7
- package/node_modules/@maka/runtime-host/dist/assistant/index-initiative/prompt.js +2 -2
- package/node_modules/@maka/runtime-host/dist/assistant/index-initiative/quick.js +16 -0
- package/node_modules/@maka/runtime-host/dist/assistant/proactive-matters/controller.js +26 -0
- package/node_modules/@maka/runtime-host/dist/assistant/run-cost.js +189 -0
- package/node_modules/@maka/runtime-host/dist/assistant/service.js +4 -0
- package/node_modules/@maka/runtime-host/dist/protocol/index.js +2 -1
- package/node_modules/@maka/runtime-host/dist/server/canonical-usage-reader.js +24 -0
- package/node_modules/@maka/runtime-host/dist/server/execution-composition.js +2 -0
- package/node_modules/@maka/runtime-host/dist/server/execution-model-authority.js +5 -0
- package/node_modules/@maka/runtime-host/dist/server/execution-model-composition.js +2 -0
- package/node_modules/@maka/runtime-host/package.json +1 -1
- package/node_modules/qs/.editorconfig +1 -1
- package/node_modules/qs/.github/THREAT_MODEL.md +4 -2
- package/node_modules/qs/CHANGELOG.md +61 -14
- package/node_modules/qs/README.md +57 -4
- package/node_modules/qs/dist/qs.js +19 -19
- package/node_modules/qs/eslint.config.mjs +61 -50
- package/node_modules/qs/lib/formats.js +7 -2
- package/node_modules/qs/lib/parse.js +105 -8
- package/node_modules/qs/lib/stringify.js +9 -7
- package/node_modules/qs/lib/utils.js +145 -13
- package/node_modules/qs/package.json +8 -8
- package/node_modules/systeminformation/lib/filesystem.js +2 -1
- package/node_modules/systeminformation/lib/graphics.js +1 -1
- package/node_modules/systeminformation/lib/index.js +10 -8
- package/node_modules/systeminformation/lib/network.js +29 -20
- package/node_modules/systeminformation/lib/processes.js +67 -79
- package/node_modules/systeminformation/lib/users.js +1 -1
- package/node_modules/systeminformation/lib/util.js +75 -30
- package/node_modules/systeminformation/package.json +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Licensed to the Apache Software Foundation (ASF) under one
|
|
3
|
+
* or more contributor license agreements. See the NOTICE file
|
|
4
|
+
* distributed with this work for additional information
|
|
5
|
+
* regarding copyright ownership. The ASF licenses this file
|
|
6
|
+
* to you under the Apache License, Version 2.0 (the
|
|
7
|
+
* "License"); you may not use this file except in compliance
|
|
8
|
+
* with the License. You may obtain a copy of the License at
|
|
9
|
+
*
|
|
10
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
*
|
|
12
|
+
* Unless required by applicable law or agreed to in writing,
|
|
13
|
+
* software distributed under the License is distributed on an
|
|
14
|
+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
15
|
+
* KIND, either express or implied. See the License for the
|
|
16
|
+
* specific language governing permissions and limitations
|
|
17
|
+
* under the License.
|
|
18
|
+
*/
|
|
19
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
20
|
+
import { mkdirSync } from 'node:fs';
|
|
21
|
+
import { join } from 'node:path';
|
|
22
|
+
const DAY_MS = 86_400_000;
|
|
23
|
+
/** Calendar day in the Host's local time zone, `YYYY-MM-DD`. */
|
|
24
|
+
export function localDay(at) {
|
|
25
|
+
const date = new Date(at);
|
|
26
|
+
const pad = (value) => String(value).padStart(2, '0');
|
|
27
|
+
return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`;
|
|
28
|
+
}
|
|
29
|
+
const UNKNOWN_USAGE = {
|
|
30
|
+
calls: 0,
|
|
31
|
+
inputTokens: 0,
|
|
32
|
+
cacheReadTokens: 0,
|
|
33
|
+
cacheWriteTokens: 0,
|
|
34
|
+
outputTokens: 0,
|
|
35
|
+
modelDurationMs: 0,
|
|
36
|
+
complete: false,
|
|
37
|
+
};
|
|
38
|
+
export class RunCostLedger {
|
|
39
|
+
readUsage;
|
|
40
|
+
now;
|
|
41
|
+
#db;
|
|
42
|
+
#closed = false;
|
|
43
|
+
constructor(directory, readUsage, now = Date.now) {
|
|
44
|
+
this.readUsage = readUsage;
|
|
45
|
+
this.now = now;
|
|
46
|
+
// A damaged observability database must not keep the assistant from
|
|
47
|
+
// starting: the ledger then records nothing and reads back empty.
|
|
48
|
+
try {
|
|
49
|
+
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
50
|
+
this.#db = new DatabaseSync(join(directory, 'run-cost.sqlite'));
|
|
51
|
+
this.#init(this.#db);
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
this.#db?.close();
|
|
55
|
+
this.#db = undefined;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
#init(db) {
|
|
59
|
+
db.exec(`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000;
|
|
60
|
+
CREATE TABLE IF NOT EXISTS run_cost(
|
|
61
|
+
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
62
|
+
kind TEXT NOT NULL,
|
|
63
|
+
run_id TEXT NOT NULL,
|
|
64
|
+
matter_id TEXT,
|
|
65
|
+
day TEXT NOT NULL,
|
|
66
|
+
started_at INTEGER NOT NULL,
|
|
67
|
+
ended_at INTEGER NOT NULL,
|
|
68
|
+
duration_ms INTEGER NOT NULL,
|
|
69
|
+
outcome TEXT NOT NULL,
|
|
70
|
+
reported INTEGER,
|
|
71
|
+
calls INTEGER NOT NULL,
|
|
72
|
+
input_tokens INTEGER NOT NULL,
|
|
73
|
+
cache_read_tokens INTEGER NOT NULL,
|
|
74
|
+
cache_write_tokens INTEGER NOT NULL,
|
|
75
|
+
output_tokens INTEGER NOT NULL,
|
|
76
|
+
model_duration_ms INTEGER NOT NULL,
|
|
77
|
+
usage_complete INTEGER NOT NULL);
|
|
78
|
+
CREATE INDEX IF NOT EXISTS run_cost_day ON run_cost(day, kind);
|
|
79
|
+
CREATE INDEX IF NOT EXISTS run_cost_matter ON run_cost(matter_id, started_at);`);
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Appends one finished run. Never throws: observing a run must not change it,
|
|
83
|
+
* so a closed ledger, an unreadable Usage ledger or a full disk loses the row
|
|
84
|
+
* and nothing else. Returns whether the row was written.
|
|
85
|
+
*/
|
|
86
|
+
async record(run) {
|
|
87
|
+
if (this.#closed || !this.#db)
|
|
88
|
+
return false;
|
|
89
|
+
try {
|
|
90
|
+
const usage = await this.#usage(run.segments);
|
|
91
|
+
if (this.#closed || !this.#db)
|
|
92
|
+
return false;
|
|
93
|
+
this.#db
|
|
94
|
+
.prepare(`INSERT INTO run_cost(kind,run_id,matter_id,day,started_at,ended_at,duration_ms,outcome,reported,
|
|
95
|
+
calls,input_tokens,cache_read_tokens,cache_write_tokens,output_tokens,model_duration_ms,usage_complete)
|
|
96
|
+
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)
|
|
97
|
+
.run(run.kind, run.runId, run.matterId ?? null, localDay(run.startedAt), run.startedAt, run.endedAt, Math.max(0, run.endedAt - run.startedAt), run.outcome, run.kind === 'matter_activation' ? null : run.outcome === 'reported' ? 1 : 0, usage.calls, usage.inputTokens, usage.cacheReadTokens, usage.cacheWriteTokens, usage.outputTokens, usage.modelDurationMs, usage.complete ? 1 : 0);
|
|
98
|
+
return true;
|
|
99
|
+
}
|
|
100
|
+
catch {
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
async #usage(segments) {
|
|
105
|
+
if (!this.readUsage || !segments.length)
|
|
106
|
+
return UNKNOWN_USAGE;
|
|
107
|
+
const sum = { ...UNKNOWN_USAGE, complete: true };
|
|
108
|
+
for (const segment of segments) {
|
|
109
|
+
let part;
|
|
110
|
+
try {
|
|
111
|
+
part = await this.readUsage(segment.sessionId, segment.from, segment.to);
|
|
112
|
+
}
|
|
113
|
+
catch {
|
|
114
|
+
// An unreadable window is unknown, not zero.
|
|
115
|
+
sum.complete = false;
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
sum.calls += part.calls;
|
|
119
|
+
sum.inputTokens += part.inputTokens;
|
|
120
|
+
sum.cacheReadTokens += part.cacheReadTokens;
|
|
121
|
+
sum.cacheWriteTokens += part.cacheWriteTokens;
|
|
122
|
+
sum.outputTokens += part.outputTokens;
|
|
123
|
+
sum.modelDurationMs += part.modelDurationMs;
|
|
124
|
+
sum.complete &&= part.complete;
|
|
125
|
+
}
|
|
126
|
+
return sum;
|
|
127
|
+
}
|
|
128
|
+
/** Per-day totals and per-Matter activations for the last `days` calendar days, plus recent runs. */
|
|
129
|
+
summary(options = {}) {
|
|
130
|
+
const db = this.#db;
|
|
131
|
+
if (this.#closed || !db)
|
|
132
|
+
return { daily: [], matters: [], recent: [] };
|
|
133
|
+
const days = Math.min(Math.max(options.days ?? 7, 1), 90);
|
|
134
|
+
const since = localDay(this.now() - (days - 1) * DAY_MS);
|
|
135
|
+
const totals = `COUNT(*) AS runs,
|
|
136
|
+
COALESCE(SUM(outcome='reported'),0) AS reported,
|
|
137
|
+
COALESCE(SUM(outcome='no_report'),0) AS noReport,
|
|
138
|
+
COALESCE(SUM(outcome='failed'),0) AS failed,
|
|
139
|
+
COALESCE(SUM(calls),0) AS calls,
|
|
140
|
+
COALESCE(SUM(input_tokens),0) AS inputTokens,
|
|
141
|
+
COALESCE(SUM(cache_read_tokens),0) AS cacheReadTokens,
|
|
142
|
+
COALESCE(SUM(cache_write_tokens),0) AS cacheWriteTokens,
|
|
143
|
+
COALESCE(SUM(output_tokens),0) AS outputTokens,
|
|
144
|
+
COALESCE(SUM(duration_ms),0) AS durationMs,
|
|
145
|
+
COALESCE(SUM(model_duration_ms),0) AS modelDurationMs,
|
|
146
|
+
COALESCE(SUM(usage_complete=0),0) AS incompleteUsageRuns`;
|
|
147
|
+
const daily = db
|
|
148
|
+
.prepare(`SELECT day, kind, ${totals} FROM run_cost WHERE day >= ? GROUP BY day, kind ORDER BY day DESC, kind`)
|
|
149
|
+
.all(since)
|
|
150
|
+
.map((row) => ({ ...row }));
|
|
151
|
+
const matters = db
|
|
152
|
+
.prepare(`SELECT matter_id AS matterId, ${totals} FROM run_cost
|
|
153
|
+
WHERE kind='matter_activation' AND matter_id IS NOT NULL AND day >= ?
|
|
154
|
+
GROUP BY matter_id ORDER BY runs DESC, matter_id`)
|
|
155
|
+
.all(since)
|
|
156
|
+
.map((row) => {
|
|
157
|
+
const { runs, reported: _reported, noReport: _noReport, ...rest } = row;
|
|
158
|
+
return { ...rest, activations: runs };
|
|
159
|
+
});
|
|
160
|
+
const recent = db
|
|
161
|
+
.prepare('SELECT * FROM run_cost ORDER BY seq DESC LIMIT ?')
|
|
162
|
+
.all(Math.min(Math.max(options.recent ?? 20, 0), 100))
|
|
163
|
+
.map((row) => ({
|
|
164
|
+
kind: row.kind,
|
|
165
|
+
runId: row.run_id,
|
|
166
|
+
matterId: row.matter_id,
|
|
167
|
+
day: row.day,
|
|
168
|
+
startedAt: row.started_at,
|
|
169
|
+
endedAt: row.ended_at,
|
|
170
|
+
durationMs: row.duration_ms,
|
|
171
|
+
outcome: row.outcome,
|
|
172
|
+
reported: row.reported === null ? null : row.reported === 1,
|
|
173
|
+
calls: row.calls,
|
|
174
|
+
inputTokens: row.input_tokens,
|
|
175
|
+
cacheReadTokens: row.cache_read_tokens,
|
|
176
|
+
cacheWriteTokens: row.cache_write_tokens,
|
|
177
|
+
outputTokens: row.output_tokens,
|
|
178
|
+
modelDurationMs: row.model_duration_ms,
|
|
179
|
+
usageComplete: row.usage_complete === 1,
|
|
180
|
+
}));
|
|
181
|
+
return { daily, matters, recent };
|
|
182
|
+
}
|
|
183
|
+
close() {
|
|
184
|
+
if (this.#closed)
|
|
185
|
+
return;
|
|
186
|
+
this.#closed = true;
|
|
187
|
+
this.#db?.close();
|
|
188
|
+
}
|
|
189
|
+
}
|
|
@@ -21,6 +21,7 @@ import { BOT_TOOLS, BOT_DELEGATION_TOOLS } from './bot/capabilities.js';
|
|
|
21
21
|
import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
|
|
22
22
|
import { join, isAbsolute } from 'node:path';
|
|
23
23
|
import { createHash, randomUUID } from 'node:crypto';
|
|
24
|
+
import { RunCostLedger } from './run-cost.js';
|
|
24
25
|
export const ASSISTANT_LEGACY_PACKAGES = [
|
|
25
26
|
'dev.maka.memory-network',
|
|
26
27
|
'dev.maka.proactive-matters',
|
|
@@ -74,8 +75,11 @@ class AssistantRuntime {
|
|
|
74
75
|
if (this.initialized)
|
|
75
76
|
throw Error('Personal assistant already initialized');
|
|
76
77
|
const settings = await this.loadSettings(legacy, entries);
|
|
78
|
+
const runCost = new RunCostLedger(this.directory, this.dependencies.readSessionUsage);
|
|
79
|
+
this.disposers.push(() => runCost.close());
|
|
77
80
|
const services = {
|
|
78
81
|
bot: this.bot,
|
|
82
|
+
runCost,
|
|
79
83
|
backgroundSessions: { register: (read) => this.backgroundSessions.push(read) },
|
|
80
84
|
...this.dependencies,
|
|
81
85
|
onClose: (dispose) => this.disposers.push(dispose),
|
|
@@ -70,7 +70,8 @@ export const RUNTIME_HOST_PROTOCOL_VERSION = 0;
|
|
|
70
70
|
// Increment when the same protocol version no longer guarantees safe Client-Host
|
|
71
71
|
// interoperability. Mismatches are rejected before domain commands are admitted.
|
|
72
72
|
// Re-check against main before merging; sibling PRs may use the same next epoch.
|
|
73
|
-
export const RUNTIME_HOST_COMPATIBILITY_EPOCH =
|
|
73
|
+
export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 237;
|
|
74
|
+
// 237: Codex model discovery carries a compaction compatibility hash.
|
|
74
75
|
// 236: Goal event wait consumers include the assistant character and pet operations from main.
|
|
75
76
|
// 235: Goal event wait consumers include the quick-discovery operations from main.
|
|
76
77
|
// 233: Goal event wait consumers include the Session check-in operations from main.
|
|
@@ -53,6 +53,30 @@ export function readCanonicalUsageLogs(stores, query, now, limit, repair = true)
|
|
|
53
53
|
// source's own first `offset + limit` rows.
|
|
54
54
|
return readCanonical(stores, query, repair, () => stores.modelCalls.modelCallLogs(query, now, 0, limit));
|
|
55
55
|
}
|
|
56
|
+
/**
|
|
57
|
+
* One Session's canonical usage in a time window, for the assistant's run-cost
|
|
58
|
+
* ledger (#6309). It is the window technique hosted-execution settlement uses:
|
|
59
|
+
* attempts are attributed by the Session that made them and the time they
|
|
60
|
+
* completed in. Anything the ledger could not fully account for (partial or
|
|
61
|
+
* missing provider usage, unreadable or unrepaired rows) marks the answer
|
|
62
|
+
* incomplete rather than presenting a lower bound as the whole.
|
|
63
|
+
*/
|
|
64
|
+
export async function readSessionUsageWindow(stores, sessionId, from, to) {
|
|
65
|
+
const source = await readCanonicalUsageSummary(stores, { range: { from, to }, sessionId }, Date.now());
|
|
66
|
+
const { totalTokens, coverage } = source.projection;
|
|
67
|
+
return {
|
|
68
|
+
calls: source.projection.totalRequests,
|
|
69
|
+
inputTokens: totalTokens.input,
|
|
70
|
+
cacheReadTokens: totalTokens.cacheRead,
|
|
71
|
+
cacheWriteTokens: totalTokens.cacheWrite,
|
|
72
|
+
outputTokens: totalTokens.output,
|
|
73
|
+
modelDurationMs: source.projection.totalDurationMs,
|
|
74
|
+
complete: source.unreadableRecords === 0 &&
|
|
75
|
+
source.pendingRepairs === 0 &&
|
|
76
|
+
coverage.usagePartialAttempts === 0 &&
|
|
77
|
+
coverage.usageMissingAttempts === 0,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
56
80
|
/** Runs one bounded repair pass and rejects data still unsafe for durable derivatives. */
|
|
57
81
|
export async function readCompleteCanonicalUsageSummary(stores, query, now) {
|
|
58
82
|
const source = await readCanonicalUsageSummary(stores, query, now);
|
|
@@ -196,6 +196,7 @@ import { SkillCatalogRepository } from './skill-catalog-repository.js';
|
|
|
196
196
|
import { HostSessionTodoCoordinator } from './session-todo-coordinator.js';
|
|
197
197
|
import { HostTurnControlCoordinator } from './turn-control-coordinator.js';
|
|
198
198
|
import { HostUsagePricingCoordinator } from './usage-pricing-coordinator.js';
|
|
199
|
+
import { readSessionUsageWindow } from './canonical-usage-reader.js';
|
|
199
200
|
import { HostWebSearchCoordinator } from './web-search-coordinator.js';
|
|
200
201
|
import { HostWorkHubCoordinationCoordinator } from './workhub-coordination-coordinator.js';
|
|
201
202
|
import { WorkHubActionEffectFailure, workHubResumedTurnId, } from './workhub-coordination-action-gate.js';
|
|
@@ -408,6 +409,7 @@ export async function createExecutionRuntimeHostComposition(context, options = {
|
|
|
408
409
|
sessionQuery: pluginSessionQuery,
|
|
409
410
|
sources: pluginSources,
|
|
410
411
|
resolveTools: (sessionId, core) => pluginTools.resolve(sessionId, core),
|
|
412
|
+
readSessionUsage: (sessionId, from, to) => readSessionUsageWindow(openedUsageStores, sessionId, from, to),
|
|
411
413
|
}));
|
|
412
414
|
const isAssistantSession = (header) => header.labels?.some((label) => label === 'personal-assistant' || label === ASSISTANT_BACKGROUND_SESSION_LABEL) ?? false;
|
|
413
415
|
const assertAssistantSessionEnabled = async (sessionId) => {
|
|
@@ -750,6 +750,11 @@ export async function resolveExecutionTarget(header, runtimePolicy, oauthCredent
|
|
|
750
750
|
? {}
|
|
751
751
|
: { defaultApiProtocol: resolved.connection.defaultApiProtocol }),
|
|
752
752
|
defaultModel: model,
|
|
753
|
+
...(resolved.connection.providerType === 'openai-codex' &&
|
|
754
|
+
resolved.connection.modelSource === 'fetched' &&
|
|
755
|
+
resolved.connection.modelsFetchedAt !== undefined
|
|
756
|
+
? { codexCompactionMetadataFetchedAt: resolved.connection.modelsFetchedAt }
|
|
757
|
+
: {}),
|
|
753
758
|
models: discovered
|
|
754
759
|
? [...resolved.connection.models]
|
|
755
760
|
: [...resolved.connection.models, modelInfo],
|
|
@@ -108,6 +108,8 @@ async function buildHostAiSdkBackend(input, target) {
|
|
|
108
108
|
connectionId: input.context.header.llmConnectionId,
|
|
109
109
|
providerStateIdentity: target.providerStateIdentity,
|
|
110
110
|
modelId: target.model,
|
|
111
|
+
models: target.connection.models,
|
|
112
|
+
codexCompactionMetadataFetchedAt: target.connection.codexCompactionMetadataFetchedAt,
|
|
111
113
|
providerOptions,
|
|
112
114
|
}), textHistorySummarizer)
|
|
113
115
|
: textHistorySummarizer;
|
|
@@ -24,7 +24,7 @@ Caller Application → qs.stringify(obj, options) → Stringifying Engine → Ou
|
|
|
24
24
|
```
|
|
25
25
|
|
|
26
26
|
**Trust Boundaries:**
|
|
27
|
-
- **Input string (parse):** May come from untrusted sources (e.g., user input, network requests)
|
|
27
|
+
- **Input string (parse):** May come from untrusted sources (e.g., user input, network requests). `qs` defines its own key grammar, where bracket groups denote nesting, so the parameter names it derives are deliberately not the flat names produced by `URLSearchParams`, `node:querystring`, or a WHATWG-conformant parser. The parsed object, not the raw string, is the only thing a caller should make trust decisions about; see "Interpretation conflict" below.
|
|
28
28
|
- **Input object (stringify):** The calling application's own data, not a direct untrusted channel. May contain cycles (handled by the cycle detector). Deeply-nested non-cyclic input surfaces as a catchable synchronous `RangeError` (stack overflow), not unbounded resource consumption; callers that serialize externally-influenced objects should bound their depth/size beforehand.
|
|
29
29
|
- **Options:** Provided by the caller
|
|
30
30
|
- **Cycle Tracking:** Used only during stringification to detect and handle circular references
|
|
@@ -51,6 +51,7 @@ Caller Application → qs.stringify(obj, options) → Stringifying Engine → Ou
|
|
|
51
51
|
- **Information Disclosure:** Error messages may expose internal details or sensitive data.
|
|
52
52
|
- **Denial of Service:** Large or malformed input can exhaust memory or CPU.
|
|
53
53
|
- **Elevation of Privilege:** Prototype pollution can lead to unintended privilege escalation in the host application[3][4][8].
|
|
54
|
+
- **Interpretation conflict (out of scope):** An application that parses the same query string with two different parsers, and makes a trust decision on one result while consuming the other, can be attacked by any input the two parsers read differently. `qs`'s bracket grammar guarantees such inputs exist, so this is a property of the deployment, not a defect `qs` can remove. It is the calling application's responsibility to parse once and validate the parsed object.
|
|
54
55
|
|
|
55
56
|
### 6. Mitigation/Countermeasures
|
|
56
57
|
|
|
@@ -58,8 +59,9 @@ Caller Application → qs.stringify(obj, options) → Stringifying Engine → Ou
|
|
|
58
59
|
|---------------------------------------------------|---------------------|
|
|
59
60
|
| Tampering (malicious input, prototype pollution) | Strict input validation; keep `allowPrototypes: false` by default; use `plainObjects` for output; ensure builtins/globals are never modified by parse[4][8]. |
|
|
60
61
|
| Information Disclosure (error messages) | Generic error messages without stack traces or internal paths. |
|
|
61
|
-
| Denial of Service (memory/CPU exhaustion) | `arrayLimit`, `parameterLimit`, and depth limiting apply with safe defaults. Note that `arrayLimit` is a representation threshold (array vs. object), and `parameterLimit` bounds only the number of `&`-delimited parameters - neither caps the elements produced by `comma: true` splitting. For
|
|
62
|
+
| Denial of Service (memory/CPU exhaustion) | `arrayLimit`, `parameterLimit`, and depth limiting apply with safe defaults. Note that `arrayLimit` is a representation threshold (array vs. object), and `parameterLimit` bounds only the number of `&`-delimited parameters - neither caps the elements produced by `comma: true` splitting. For hard caps on the number of parameters and on each array's length, opt in to `throwOnLimitExceeded: true` (default `false`), which rejects a query string with more than `parameterLimit` parameters, or one that would produce any array longer than `arrayLimit`. `arrayLimit` applies to each array separately, including each parameter's comma-split value, so the total number of values can still reach `parameterLimit` times `arrayLimit`; always bound input size at the transport layer as well[7]. On the `stringify` side, the input is the caller's own object rather than an untrusted string, so unbounded nesting manifests as a catchable `RangeError` (stack overflow) rather than exhaustion; callers serializing externally-influenced objects should bound depth/size beforehand, and may opt into `stringify`'s `depth` option (default `Infinity`) for a deterministic, catchable error. |
|
|
62
63
|
| Elevation of Privilege (prototype pollution) | Keep `allowPrototypes: false`; validate options against allowlist; use `plainObjects` to avoid prototype pollution[4][8]. |
|
|
64
|
+
| Interpretation conflict (validate-then-use across two parsers) | None available inside `qs`, by construction: `qs` and a flat parser disagree about parameter names for well-formed input, so no hardening can make the two views equal. Parse untrusted input once, with `qs`, and apply deny lists, authorization, and signature checks to the parsed object rather than to raw key text. Text that `qs` cannot place in its grammar is currently dropped, so distinct raw names can share one parsed path ([#619](https://github.com/ljharb/qs/issues/619)). |
|
|
63
65
|
|
|
64
66
|
### 7. Risk Ranking
|
|
65
67
|
|
|
@@ -1,17 +1,64 @@
|
|
|
1
|
+
## **6.17.0**
|
|
2
|
+
- [New] `stringify`: add `WHATWG` format (#623)
|
|
3
|
+
- [Fix] `stringify`: do not sort the caller's `filter` array in place (#637)
|
|
4
|
+
- [Fix] `parse`: copy an own `__proto__` key as data (#636)
|
|
5
|
+
- [Fix] `parse`: decode percent-encoded dots case-insensitively (#635)
|
|
6
|
+
- [Fix] `stringify`: encode keys when `allowEmptyArrays` is set (#561)
|
|
7
|
+
- [Fix] `parse`: preserve comma arrays when interpreting numeric entities (#630)
|
|
8
|
+
- [Fix] `parse`: allow scalar fields with a zero array limit (#627)
|
|
9
|
+
- [Fix] `parse`: do not count an existing comma array toward `arrayLimit` when `duplicates` discards or replaces it
|
|
10
|
+
- [Fix] `utils`: `encode`: encode lone surrogates as U+FFFD instead of consuming the next code unit (#624)
|
|
11
|
+
- [eslint] fix indentation
|
|
12
|
+
- [readme] document the encoder's `format` argument
|
|
13
|
+
- [meta] add changelog acknowledgement
|
|
14
|
+
- [meta] fix changelog indents
|
|
15
|
+
- [Dev Deps] update `eslint`, `evalmd`, `npmignore`
|
|
16
|
+
- [Tests] `stringify`: pin `allowEmptyArrays` output for keys that need no encoding
|
|
17
|
+
- [Tests] `stringify`: pin `sort` order for nested keys selected by an array `filter`
|
|
18
|
+
- [Tests] `parse`: cover `interpretNumericEntities` across value shapes, keys, and the charset sentinel
|
|
19
|
+
- [Tests] `parse`: cover more input that builds no array at a zero or negative `arrayLimit`
|
|
20
|
+
- [Tests] `parse`: cover `comma` with a zero or negative `arrayLimit`, and combining a full comma array
|
|
21
|
+
- [Tests] `parse`: pin the behavior of a zero or negative `arrayLimit`
|
|
22
|
+
- [Tests] `utils`/`stringify`: pin `~`, `*`, and parens encoding in RFC3986 and RFC1738
|
|
23
|
+
|
|
24
|
+
## **6.16.1**
|
|
25
|
+
- [Fix] `parse`: do not mutate an object input
|
|
26
|
+
- [Fix] `stringify`: stop re-encoding structural separator dots when `encodeDotInKeys` is set (#564)
|
|
27
|
+
- [Fix] `stringify`: `arrayFormat: 'comma'`: pass the full encoder arguments with `encodeValuesOnly`
|
|
28
|
+
- [Fix] `utils`: encode `+` as `%2B` in iso-8859-1 mode so values round-trip (#565)
|
|
29
|
+
- [Fix] parse: only convert top-level dots to brackets when allowDots is on (#621)
|
|
30
|
+
- [Fix] `stringify`: ignore inherited array format names (#622)
|
|
31
|
+
- [Fix] `stringify`: `arrayFormat: 'comma'`: do not throw on elements with no callable `toString`
|
|
32
|
+
- [Fix] `utils`: `merge`: seed an overflow container's index from its own keys, not just the source
|
|
33
|
+
- [Fix] `utils`: `merge`: keep a `strictMerge` conflict container's index and `arrayLimit` correct
|
|
34
|
+
- [Fix] `parse`: only treat `[]` as array syntax when it ends the key
|
|
35
|
+
- [Perf] `decode`: skip decoders when there is no percent sign (#583)
|
|
36
|
+
- [Robustness] `utils`: `merge`: apply `arrayLimit` to the two-element array built for colliding values
|
|
37
|
+
- [Robustness] `parse`: bound the nesting added by repeated `strictMerge` conflicts (credit @Yeaseen)
|
|
38
|
+
- [Docs] document that `qs` and flat parsers disagree about parameter names
|
|
39
|
+
- [Docs] clarify that `throwOnLimitExceeded` limits each array, not the total number of values
|
|
40
|
+
- [Docs] clarify 6.14.1/6.15.2 parse bracket behavior change (#567, #568)
|
|
41
|
+
- [meta] fix changelog indents
|
|
42
|
+
- [actions] set least-privilege `cache-mode`
|
|
43
|
+
- [Tests] `utils`: `decode`: add missing cases
|
|
44
|
+
- [Tests] `parse`: characterize the `]=` split when an `=` precedes any bracket group (#620)
|
|
45
|
+
- [Tests] `parse`: `allowDots`: cover stray close brackets before a dot
|
|
46
|
+
- [Dev Deps] update `@ljharb/eslint-config`, `eslint`, `tape`
|
|
47
|
+
|
|
1
48
|
## **6.16.0**
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
49
|
+
- [New] `stringify`: add a `depth` option to bound recursion depth (default `Infinity`)
|
|
50
|
+
- [Fix] stringify: serialize Date values when a filter is provided
|
|
51
|
+
- [Fix] `parse`: enforce `arrayLimit` on comma groups under `[]=` when `throwOnLimitExceeded` is set
|
|
52
|
+
- [Fix] `parse`: flatten a collection appended to an overflowed array (#571)
|
|
53
|
+
- [Fix] `utils`: `isBuffer`: do not invoke a non-callable `constructor.isBuffer`
|
|
54
|
+
- [Fix] `stringify`: do not let `allowEmptyArrays` skip cycle detection (or drop own keys) on an empty array with own properties
|
|
55
|
+
- [Fix] `stringify`: encode dots in a top-level key with a primitive value when encodeDotInKeys is set (#562)
|
|
56
|
+
- [Docs] threat model: clarify `stringify` deep-nesting DoS is caller-bounded
|
|
57
|
+
- [Docs] clarify `arrayLimit` is a representation threshold, not an element-count cap
|
|
58
|
+
- [Tests] `parse`: remove a test that pinned `[]=` comma groups escaping `arrayLimit`
|
|
59
|
+
- [Tests] `stringify`: pin current `encodeDotInKeys` separator-dot behavior
|
|
60
|
+
- [Dev Deps] update `@ljharb/eslint-config`, `eslint`
|
|
61
|
+
- [Dev Deps] update `eslint`, `evalmd`
|
|
15
62
|
|
|
16
63
|
## **6.15.3**
|
|
17
64
|
- [Fix] `parse`: enforce `throwOnLimitExceeded` for cumulative array growth via `combine`/`merge`
|
|
@@ -29,7 +76,7 @@
|
|
|
29
76
|
- [Fix] `stringify`: use configured `delimiter` after `charsetSentinel` (#555)
|
|
30
77
|
- [Fix] `stringify`: apply `formatter` to encoded key under `strictNullHandling` (#554)
|
|
31
78
|
- [Fix] `stringify`: skip null/undefined filter-array entries instead of crashing in `encoder` (#551)
|
|
32
|
-
- [Fix] `parse`: handle nested bracket groups and add regression tests (#530)
|
|
79
|
+
- [Fix] `parse`: handle nested bracket groups and add regression tests (#530); changes output for some unbalanced bracket keys (see #558)
|
|
33
80
|
- [readme] fix grammar (#550)
|
|
34
81
|
- [Dev Deps] update `@ljharb/eslint-config`
|
|
35
82
|
- [Tests] add regression tests for keys containing percent-encoded bracket text
|
|
@@ -85,6 +85,23 @@ assert.deepEqual(qs.parse('foo[bar][baz]=foobarbaz'), {
|
|
|
85
85
|
});
|
|
86
86
|
```
|
|
87
87
|
|
|
88
|
+
Since v6.14.1 and v6.15.2, `parse` interprets nested bracket groups more consistently. Query strings with unbalanced `[`/`]` can produce different results than in 6.15.1; see [#558](https://github.com/ljharb/qs/issues/558).
|
|
89
|
+
|
|
90
|
+
Because of that bracket grammar,
|
|
91
|
+
**qs** does not agree with `URLSearchParams`,
|
|
92
|
+
`node:querystring`, or any other flat parser about what the "name" of a parameter is,
|
|
93
|
+
and it never has.
|
|
94
|
+
`URLSearchParams` reads `'foo[bar]=baz'` as one flat name `foo[bar]`;
|
|
95
|
+
**qs** reads it as the path `foo.bar`.
|
|
96
|
+
Text that **qs**'s grammar cannot place,
|
|
97
|
+
such as the `extra` in `'foo[bar]extra=baz'`,
|
|
98
|
+
is currently dropped, so several distinct flat names can resolve to the same **qs** path
|
|
99
|
+
(see [#619](https://github.com/ljharb/qs/issues/619)).
|
|
100
|
+
|
|
101
|
+
So validating, authorizing, filtering,
|
|
102
|
+
or signing a query string with one parser and then consuming it with another is unsafe: a deny list or signature computed over `URLSearchParams` names does not constrain what **qs** will produce from the same string.
|
|
103
|
+
Parse the input once, with **qs**, and apply your checks to the resulting object.
|
|
104
|
+
|
|
88
105
|
By default, when nesting objects **qs** will only parse up to 5 children deep.
|
|
89
106
|
This means if you attempt to parse a string like `'a[b][c][d][e][f][g][h][i]=j'` your resulting object will be:
|
|
90
107
|
|
|
@@ -135,7 +152,7 @@ var limited = qs.parse('a=b&c=d', { parameterLimit: 1 });
|
|
|
135
152
|
assert.deepEqual(limited, { a: 'b' });
|
|
136
153
|
```
|
|
137
154
|
|
|
138
|
-
If you want an error to be thrown whenever
|
|
155
|
+
If you want an error to be thrown whenever a limit is exceeded (eg, `parameterLimit`, `arrayLimit`), set the `throwOnLimitExceeded` option to `true`. This option will generate a descriptive error if the query string exceeds a configured limit.
|
|
139
156
|
```javascript
|
|
140
157
|
try {
|
|
141
158
|
qs.parse('a=1&b=2&c=3&d=4', { parameterLimit: 3, throwOnLimitExceeded: true });
|
|
@@ -147,7 +164,7 @@ try {
|
|
|
147
164
|
|
|
148
165
|
When `throwOnLimitExceeded` is set to `false` (default), **qs** will parse up to the specified `parameterLimit` and ignore the rest without throwing an error.
|
|
149
166
|
|
|
150
|
-
Note that `parameterLimit` only bounds the number of `&`-delimited parameters; it does not limit how many values a single parameter expands into. In particular, when `comma: true` is enabled, a single parameter's value is split on commas into arbitrarily many elements, which `parameterLimit` does not constrain.
|
|
167
|
+
Note that `parameterLimit` only bounds the number of `&`-delimited parameters; it does not limit how many values a single parameter expands into. In particular, when `comma: true` is enabled, a single parameter's value is split on commas into arbitrarily many elements, which `parameterLimit` does not constrain. Setting `throwOnLimitExceeded: true` (see `arrayLimit` below) makes **qs** reject a query string with more than `parameterLimit` parameters, one that uses an array index of `arrayLimit` or greater, or one that would produce any array longer than `arrayLimit`, including an array produced by comma splitting. `arrayLimit` applies to each array separately, including each parameter's comma-split value, so the total number of values across all parameters can still reach `parameterLimit` times `arrayLimit`, which is 20,000 at the default limits; always bound the input size at the transport layer as well (e.g. an HTTP body-size limit).
|
|
151
168
|
|
|
152
169
|
To bypass the leading question mark, use `ignoreQueryPrefix`:
|
|
153
170
|
|
|
@@ -293,6 +310,8 @@ assert.deepEqual(withIndexedEmptyString, { a: ['b', '', 'c'] });
|
|
|
293
310
|
Any array members with an index of `20` or greater will instead be converted to an object with the index as the key.
|
|
294
311
|
This is needed to handle cases when someone sent, for example, `a[999999999]` and it will take significant time to iterate over this huge array.
|
|
295
312
|
|
|
313
|
+
Since v6.14.1 and v6.15.2, the arrayLimit now applies to arrays specified using both the `[]` and `[0]` syntaxes.
|
|
314
|
+
|
|
296
315
|
```javascript
|
|
297
316
|
var withMaxIndex = qs.parse('a[100]=b');
|
|
298
317
|
assert.deepEqual(withMaxIndex, { a: { '100': 'b' } });
|
|
@@ -317,7 +336,7 @@ try {
|
|
|
317
336
|
|
|
318
337
|
When `throwOnLimitExceeded` is set to `false` (default), **qs** will parse up to the specified `arrayLimit` and if the limit is exceeded, the array will instead be converted to an object with the index as the key.
|
|
319
338
|
|
|
320
|
-
Note that `arrayLimit` is a *representation* threshold that controls when a numerically-indexed collection switches from an array to an object
|
|
339
|
+
Note that `arrayLimit` is a *representation* threshold that controls when a numerically-indexed collection switches from an array to an object; it is **not** a hard cap on the total number of elements parsed. With the default `throwOnLimitExceeded: false`, exceeding `arrayLimit` never rejects or truncates input; it only changes the container type, and the resulting object still holds every element (so its size stays proportional to the input). This conversion is itself a safeguard: it avoids allocating a huge sparse array for input like `a[999999999]`. If you need a hard limit that rejects oversized arrays in untrusted query strings, set `throwOnLimitExceeded: true`. `throwOnLimitExceeded` still limits each array separately, not the total: with `comma: true`, each comma-separated group under `a[]=` counts as one element of the outer array, and can itself hold up to `arrayLimit` elements.
|
|
321
340
|
|
|
322
341
|
To prevent array syntax (`a[]`, `a[0]`) from being parsed as arrays, set `parseArrays` to `false`.
|
|
323
342
|
Note that duplicate keys (e.g. `a=b&a=c`) may still produce arrays when `duplicates` is `'combine'` (the default).
|
|
@@ -361,6 +380,15 @@ assert.deepEqual(arraysOfObjects, { a: ['b', 'c'] })
|
|
|
361
380
|
```
|
|
362
381
|
(_this cannot convert nested objects, such as `a={b:1},{c:d}`_)
|
|
363
382
|
|
|
383
|
+
With `[]` or index notation, the values from each comma-separated group stay together as one element of the array, while repeated plain keys flatten into one array:
|
|
384
|
+
```javascript
|
|
385
|
+
var nestedGroups = qs.parse('a[]=b,c&a[]=d', { comma: true });
|
|
386
|
+
assert.deepEqual(nestedGroups, { a: [['b', 'c'], 'd'] });
|
|
387
|
+
|
|
388
|
+
var flatGroups = qs.parse('a=b,c&a=d', { comma: true });
|
|
389
|
+
assert.deepEqual(flatGroups, { a: ['b', 'c', 'd'] });
|
|
390
|
+
```
|
|
391
|
+
|
|
364
392
|
### Parsing primitive/scalar values (numbers, booleans, null, etc)
|
|
365
393
|
|
|
366
394
|
By default, all values are parsed as strings.
|
|
@@ -449,6 +477,20 @@ var encoded = qs.stringify({ a: { b: 'c' } }, { encoder: function (str, defaultE
|
|
|
449
477
|
}})
|
|
450
478
|
```
|
|
451
479
|
|
|
480
|
+
The encoder also receives the `format` option as a fifth argument.
|
|
481
|
+
An encoder that wraps `defaultEncoder` should pass all of its arguments through,
|
|
482
|
+
since the default encoding depends on `format` (for example, RFC1738 leaves parentheses unencoded):
|
|
483
|
+
|
|
484
|
+
```javascript
|
|
485
|
+
var wrapped = qs.stringify({ a: 'b(c)' }, {
|
|
486
|
+
format: 'RFC1738',
|
|
487
|
+
encoder: function (str, defaultEncoder, charset, type, format) {
|
|
488
|
+
return defaultEncoder(str, defaultEncoder, charset, type, format);
|
|
489
|
+
}
|
|
490
|
+
});
|
|
491
|
+
assert.equal(wrapped, 'a=b(c)');
|
|
492
|
+
```
|
|
493
|
+
|
|
452
494
|
The type argument is also provided to the decoder:
|
|
453
495
|
|
|
454
496
|
```javascript
|
|
@@ -730,7 +772,8 @@ var obj = qs.parse('a=%82%B1%82%F1%82%C9%82%BF%82%CD%81I', { decoder: decoder })
|
|
|
730
772
|
assert.deepEqual(obj, { a: 'こんにちは!' });
|
|
731
773
|
```
|
|
732
774
|
|
|
733
|
-
|
|
775
|
+
<a name="rfc-3986-and-rfc-1738-space-encoding"></a>
|
|
776
|
+
### Encoding formats: RFC 3986, RFC 1738, and WHATWG
|
|
734
777
|
|
|
735
778
|
RFC3986 used as default option and encodes ' ' to *%20* which is backward compatible.
|
|
736
779
|
In the same time, output can be stringified as per RFC1738 with ' ' equal to '+'.
|
|
@@ -741,6 +784,16 @@ assert.equal(qs.stringify({ a: 'b c' }, { format : 'RFC3986' }), 'a=b%20c');
|
|
|
741
784
|
assert.equal(qs.stringify({ a: 'b c' }, { format : 'RFC1738' }), 'a=b+c');
|
|
742
785
|
```
|
|
743
786
|
|
|
787
|
+
To percent-encode keys and values the same way as the [WHATWG `application/x-www-form-urlencoded` serializer](https://url.spec.whatwg.org/#concept-urlencoded-serializer) (what `URLSearchParams` and HTML form submission use), use the `WHATWG` format.
|
|
788
|
+
Like RFC1738 it encodes ' ' as '+'; it also leaves `*` unencoded, but percent-encodes `~`, `(`, and `)`.
|
|
789
|
+
It only affects encoding; how `null`, dates, arrays, and objects are serialized is unchanged:
|
|
790
|
+
|
|
791
|
+
```javascript
|
|
792
|
+
assert.equal(qs.stringify({ a: 'b c~*()' }, { format: 'WHATWG' }), 'a=b+c%7E*%28%29');
|
|
793
|
+
```
|
|
794
|
+
|
|
795
|
+
_(Note: the `WHATWG` format matches `URLSearchParams` encoding only with the default `utf-8` charset, and only when keys are encoded too, so not with `encodeValuesOnly`. With `charset: 'iso-8859-1'`, every format uses the same `escape`-based encoding, which leaves characters like `@` and `/` unencoded; only the space handling differs)_
|
|
796
|
+
|
|
744
797
|
## Security
|
|
745
798
|
|
|
746
799
|
Please email [@ljharb](https://github.com/ljharb) or see https://tidelift.com/security if you have a potential security vulnerability to report.
|