maka-agent 0.2.2-dev.101.20261026 → 0.2.2-dev.105.20261029

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/THIRD_PARTY_NOTICES.txt +2 -2
  2. package/native/runtime-host-peer/prebuilds/win32-x64/maka_runtime_host_peer.node +0 -0
  3. package/native/runtime-host-windows-task-launcher/prebuilds/win32-x64/maka-runtime-host-task-launcher.exe +0 -0
  4. package/node_modules/@maka/core/dist/runtime-policy/connection-catalog-codec.js +6 -0
  5. package/node_modules/@maka/runtime/dist/ai-sdk-turn.js +21 -7
  6. package/node_modules/@maka/runtime/dist/history-compact-checkpoint.js +32 -4
  7. package/node_modules/@maka/runtime/dist/model-fetcher.js +7 -0
  8. package/node_modules/@maka/runtime/dist/openai-codex-history-compactor.js +13 -4
  9. package/node_modules/@maka/runtime-host/dist/assistant/index-initiative/assistant.js +3 -0
  10. package/node_modules/@maka/runtime-host/dist/assistant/index-initiative/controller.js +27 -2
  11. package/node_modules/@maka/runtime-host/dist/assistant/index-initiative/notebook.js +73 -7
  12. package/node_modules/@maka/runtime-host/dist/assistant/index-initiative/prompt.js +2 -2
  13. package/node_modules/@maka/runtime-host/dist/assistant/index-initiative/quick.js +16 -0
  14. package/node_modules/@maka/runtime-host/dist/assistant/proactive-matters/controller.js +26 -0
  15. package/node_modules/@maka/runtime-host/dist/assistant/run-cost.js +189 -0
  16. package/node_modules/@maka/runtime-host/dist/assistant/service.js +4 -0
  17. package/node_modules/@maka/runtime-host/dist/protocol/index.js +2 -1
  18. package/node_modules/@maka/runtime-host/dist/server/canonical-usage-reader.js +24 -0
  19. package/node_modules/@maka/runtime-host/dist/server/execution-composition.js +2 -0
  20. package/node_modules/@maka/runtime-host/dist/server/execution-model-authority.js +5 -0
  21. package/node_modules/@maka/runtime-host/dist/server/execution-model-composition.js +2 -0
  22. package/node_modules/@maka/runtime-host/package.json +1 -1
  23. package/node_modules/qs/.editorconfig +1 -1
  24. package/node_modules/qs/.github/THREAT_MODEL.md +4 -2
  25. package/node_modules/qs/CHANGELOG.md +61 -14
  26. package/node_modules/qs/README.md +57 -4
  27. package/node_modules/qs/dist/qs.js +19 -19
  28. package/node_modules/qs/eslint.config.mjs +61 -50
  29. package/node_modules/qs/lib/formats.js +7 -2
  30. package/node_modules/qs/lib/parse.js +105 -8
  31. package/node_modules/qs/lib/stringify.js +9 -7
  32. package/node_modules/qs/lib/utils.js +145 -13
  33. package/node_modules/qs/package.json +8 -8
  34. package/node_modules/systeminformation/lib/filesystem.js +2 -1
  35. package/node_modules/systeminformation/lib/graphics.js +1 -1
  36. package/node_modules/systeminformation/lib/index.js +10 -8
  37. package/node_modules/systeminformation/lib/network.js +29 -20
  38. package/node_modules/systeminformation/lib/processes.js +67 -79
  39. package/node_modules/systeminformation/lib/users.js +1 -1
  40. package/node_modules/systeminformation/lib/util.js +75 -30
  41. package/node_modules/systeminformation/package.json +1 -1
  42. package/package.json +1 -1
@@ -0,0 +1,189 @@
1
+ /*
2
+ * Licensed to the Apache Software Foundation (ASF) under one
3
+ * or more contributor license agreements. See the NOTICE file
4
+ * distributed with this work for additional information
5
+ * regarding copyright ownership. The ASF licenses this file
6
+ * to you under the Apache License, Version 2.0 (the
7
+ * "License"); you may not use this file except in compliance
8
+ * with the License. You may obtain a copy of the License at
9
+ *
10
+ * http://www.apache.org/licenses/LICENSE-2.0
11
+ *
12
+ * Unless required by applicable law or agreed to in writing,
13
+ * software distributed under the License is distributed on an
14
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
+ * KIND, either express or implied. See the License for the
16
+ * specific language governing permissions and limitations
17
+ * under the License.
18
+ */
19
+ import { DatabaseSync } from 'node:sqlite';
20
+ import { mkdirSync } from 'node:fs';
21
+ import { join } from 'node:path';
22
+ const DAY_MS = 86_400_000;
23
+ /** Calendar day in the Host's local time zone, `YYYY-MM-DD`. */
24
+ export function localDay(at) {
25
+ const date = new Date(at);
26
+ const pad = (value) => String(value).padStart(2, '0');
27
+ return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`;
28
+ }
29
+ const UNKNOWN_USAGE = {
30
+ calls: 0,
31
+ inputTokens: 0,
32
+ cacheReadTokens: 0,
33
+ cacheWriteTokens: 0,
34
+ outputTokens: 0,
35
+ modelDurationMs: 0,
36
+ complete: false,
37
+ };
38
+ export class RunCostLedger {
39
+ readUsage;
40
+ now;
41
+ #db;
42
+ #closed = false;
43
+ constructor(directory, readUsage, now = Date.now) {
44
+ this.readUsage = readUsage;
45
+ this.now = now;
46
+ // A damaged observability database must not keep the assistant from
47
+ // starting: the ledger then records nothing and reads back empty.
48
+ try {
49
+ mkdirSync(directory, { recursive: true, mode: 0o700 });
50
+ this.#db = new DatabaseSync(join(directory, 'run-cost.sqlite'));
51
+ this.#init(this.#db);
52
+ }
53
+ catch {
54
+ this.#db?.close();
55
+ this.#db = undefined;
56
+ }
57
+ }
58
+ #init(db) {
59
+ db.exec(`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000;
60
+ CREATE TABLE IF NOT EXISTS run_cost(
61
+ seq INTEGER PRIMARY KEY AUTOINCREMENT,
62
+ kind TEXT NOT NULL,
63
+ run_id TEXT NOT NULL,
64
+ matter_id TEXT,
65
+ day TEXT NOT NULL,
66
+ started_at INTEGER NOT NULL,
67
+ ended_at INTEGER NOT NULL,
68
+ duration_ms INTEGER NOT NULL,
69
+ outcome TEXT NOT NULL,
70
+ reported INTEGER,
71
+ calls INTEGER NOT NULL,
72
+ input_tokens INTEGER NOT NULL,
73
+ cache_read_tokens INTEGER NOT NULL,
74
+ cache_write_tokens INTEGER NOT NULL,
75
+ output_tokens INTEGER NOT NULL,
76
+ model_duration_ms INTEGER NOT NULL,
77
+ usage_complete INTEGER NOT NULL);
78
+ CREATE INDEX IF NOT EXISTS run_cost_day ON run_cost(day, kind);
79
+ CREATE INDEX IF NOT EXISTS run_cost_matter ON run_cost(matter_id, started_at);`);
80
+ }
81
+ /**
82
+ * Appends one finished run. Never throws: observing a run must not change it,
83
+ * so a closed ledger, an unreadable Usage ledger or a full disk loses the row
84
+ * and nothing else. Returns whether the row was written.
85
+ */
86
+ async record(run) {
87
+ if (this.#closed || !this.#db)
88
+ return false;
89
+ try {
90
+ const usage = await this.#usage(run.segments);
91
+ if (this.#closed || !this.#db)
92
+ return false;
93
+ this.#db
94
+ .prepare(`INSERT INTO run_cost(kind,run_id,matter_id,day,started_at,ended_at,duration_ms,outcome,reported,
95
+ calls,input_tokens,cache_read_tokens,cache_write_tokens,output_tokens,model_duration_ms,usage_complete)
96
+ VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)
97
+ .run(run.kind, run.runId, run.matterId ?? null, localDay(run.startedAt), run.startedAt, run.endedAt, Math.max(0, run.endedAt - run.startedAt), run.outcome, run.kind === 'matter_activation' ? null : run.outcome === 'reported' ? 1 : 0, usage.calls, usage.inputTokens, usage.cacheReadTokens, usage.cacheWriteTokens, usage.outputTokens, usage.modelDurationMs, usage.complete ? 1 : 0);
98
+ return true;
99
+ }
100
+ catch {
101
+ return false;
102
+ }
103
+ }
104
+ async #usage(segments) {
105
+ if (!this.readUsage || !segments.length)
106
+ return UNKNOWN_USAGE;
107
+ const sum = { ...UNKNOWN_USAGE, complete: true };
108
+ for (const segment of segments) {
109
+ let part;
110
+ try {
111
+ part = await this.readUsage(segment.sessionId, segment.from, segment.to);
112
+ }
113
+ catch {
114
+ // An unreadable window is unknown, not zero.
115
+ sum.complete = false;
116
+ continue;
117
+ }
118
+ sum.calls += part.calls;
119
+ sum.inputTokens += part.inputTokens;
120
+ sum.cacheReadTokens += part.cacheReadTokens;
121
+ sum.cacheWriteTokens += part.cacheWriteTokens;
122
+ sum.outputTokens += part.outputTokens;
123
+ sum.modelDurationMs += part.modelDurationMs;
124
+ sum.complete &&= part.complete;
125
+ }
126
+ return sum;
127
+ }
128
+ /** Per-day totals and per-Matter activations for the last `days` calendar days, plus recent runs. */
129
+ summary(options = {}) {
130
+ const db = this.#db;
131
+ if (this.#closed || !db)
132
+ return { daily: [], matters: [], recent: [] };
133
+ const days = Math.min(Math.max(options.days ?? 7, 1), 90);
134
+ const since = localDay(this.now() - (days - 1) * DAY_MS);
135
+ const totals = `COUNT(*) AS runs,
136
+ COALESCE(SUM(outcome='reported'),0) AS reported,
137
+ COALESCE(SUM(outcome='no_report'),0) AS noReport,
138
+ COALESCE(SUM(outcome='failed'),0) AS failed,
139
+ COALESCE(SUM(calls),0) AS calls,
140
+ COALESCE(SUM(input_tokens),0) AS inputTokens,
141
+ COALESCE(SUM(cache_read_tokens),0) AS cacheReadTokens,
142
+ COALESCE(SUM(cache_write_tokens),0) AS cacheWriteTokens,
143
+ COALESCE(SUM(output_tokens),0) AS outputTokens,
144
+ COALESCE(SUM(duration_ms),0) AS durationMs,
145
+ COALESCE(SUM(model_duration_ms),0) AS modelDurationMs,
146
+ COALESCE(SUM(usage_complete=0),0) AS incompleteUsageRuns`;
147
+ const daily = db
148
+ .prepare(`SELECT day, kind, ${totals} FROM run_cost WHERE day >= ? GROUP BY day, kind ORDER BY day DESC, kind`)
149
+ .all(since)
150
+ .map((row) => ({ ...row }));
151
+ const matters = db
152
+ .prepare(`SELECT matter_id AS matterId, ${totals} FROM run_cost
153
+ WHERE kind='matter_activation' AND matter_id IS NOT NULL AND day >= ?
154
+ GROUP BY matter_id ORDER BY runs DESC, matter_id`)
155
+ .all(since)
156
+ .map((row) => {
157
+ const { runs, reported: _reported, noReport: _noReport, ...rest } = row;
158
+ return { ...rest, activations: runs };
159
+ });
160
+ const recent = db
161
+ .prepare('SELECT * FROM run_cost ORDER BY seq DESC LIMIT ?')
162
+ .all(Math.min(Math.max(options.recent ?? 20, 0), 100))
163
+ .map((row) => ({
164
+ kind: row.kind,
165
+ runId: row.run_id,
166
+ matterId: row.matter_id,
167
+ day: row.day,
168
+ startedAt: row.started_at,
169
+ endedAt: row.ended_at,
170
+ durationMs: row.duration_ms,
171
+ outcome: row.outcome,
172
+ reported: row.reported === null ? null : row.reported === 1,
173
+ calls: row.calls,
174
+ inputTokens: row.input_tokens,
175
+ cacheReadTokens: row.cache_read_tokens,
176
+ cacheWriteTokens: row.cache_write_tokens,
177
+ outputTokens: row.output_tokens,
178
+ modelDurationMs: row.model_duration_ms,
179
+ usageComplete: row.usage_complete === 1,
180
+ }));
181
+ return { daily, matters, recent };
182
+ }
183
+ close() {
184
+ if (this.#closed)
185
+ return;
186
+ this.#closed = true;
187
+ this.#db?.close();
188
+ }
189
+ }
@@ -21,6 +21,7 @@ import { BOT_TOOLS, BOT_DELEGATION_TOOLS } from './bot/capabilities.js';
21
21
  import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
22
22
  import { join, isAbsolute } from 'node:path';
23
23
  import { createHash, randomUUID } from 'node:crypto';
24
+ import { RunCostLedger } from './run-cost.js';
24
25
  export const ASSISTANT_LEGACY_PACKAGES = [
25
26
  'dev.maka.memory-network',
26
27
  'dev.maka.proactive-matters',
@@ -74,8 +75,11 @@ class AssistantRuntime {
74
75
  if (this.initialized)
75
76
  throw Error('Personal assistant already initialized');
76
77
  const settings = await this.loadSettings(legacy, entries);
78
+ const runCost = new RunCostLedger(this.directory, this.dependencies.readSessionUsage);
79
+ this.disposers.push(() => runCost.close());
77
80
  const services = {
78
81
  bot: this.bot,
82
+ runCost,
79
83
  backgroundSessions: { register: (read) => this.backgroundSessions.push(read) },
80
84
  ...this.dependencies,
81
85
  onClose: (dispose) => this.disposers.push(dispose),
@@ -70,7 +70,8 @@ export const RUNTIME_HOST_PROTOCOL_VERSION = 0;
70
70
  // Increment when the same protocol version no longer guarantees safe Client-Host
71
71
  // interoperability. Mismatches are rejected before domain commands are admitted.
72
72
  // Re-check against main before merging; sibling PRs may use the same next epoch.
73
- export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 236;
73
+ export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 237;
74
+ // 237: Codex model discovery carries a compaction compatibility hash.
74
75
  // 236: Goal event wait consumers include the assistant character and pet operations from main.
75
76
  // 235: Goal event wait consumers include the quick-discovery operations from main.
76
77
  // 233: Goal event wait consumers include the Session check-in operations from main.
@@ -53,6 +53,30 @@ export function readCanonicalUsageLogs(stores, query, now, limit, repair = true)
53
53
  // source's own first `offset + limit` rows.
54
54
  return readCanonical(stores, query, repair, () => stores.modelCalls.modelCallLogs(query, now, 0, limit));
55
55
  }
56
+ /**
57
+ * One Session's canonical usage in a time window, for the assistant's run-cost
58
+ * ledger (#6309). It is the window technique hosted-execution settlement uses:
59
+ * attempts are attributed by the Session that made them and the time they
60
+ * completed in. Anything the ledger could not fully account for (partial or
61
+ * missing provider usage, unreadable or unrepaired rows) marks the answer
62
+ * incomplete rather than presenting a lower bound as the whole.
63
+ */
64
+ export async function readSessionUsageWindow(stores, sessionId, from, to) {
65
+ const source = await readCanonicalUsageSummary(stores, { range: { from, to }, sessionId }, Date.now());
66
+ const { totalTokens, coverage } = source.projection;
67
+ return {
68
+ calls: source.projection.totalRequests,
69
+ inputTokens: totalTokens.input,
70
+ cacheReadTokens: totalTokens.cacheRead,
71
+ cacheWriteTokens: totalTokens.cacheWrite,
72
+ outputTokens: totalTokens.output,
73
+ modelDurationMs: source.projection.totalDurationMs,
74
+ complete: source.unreadableRecords === 0 &&
75
+ source.pendingRepairs === 0 &&
76
+ coverage.usagePartialAttempts === 0 &&
77
+ coverage.usageMissingAttempts === 0,
78
+ };
79
+ }
56
80
  /** Runs one bounded repair pass and rejects data still unsafe for durable derivatives. */
57
81
  export async function readCompleteCanonicalUsageSummary(stores, query, now) {
58
82
  const source = await readCanonicalUsageSummary(stores, query, now);
@@ -196,6 +196,7 @@ import { SkillCatalogRepository } from './skill-catalog-repository.js';
196
196
  import { HostSessionTodoCoordinator } from './session-todo-coordinator.js';
197
197
  import { HostTurnControlCoordinator } from './turn-control-coordinator.js';
198
198
  import { HostUsagePricingCoordinator } from './usage-pricing-coordinator.js';
199
+ import { readSessionUsageWindow } from './canonical-usage-reader.js';
199
200
  import { HostWebSearchCoordinator } from './web-search-coordinator.js';
200
201
  import { HostWorkHubCoordinationCoordinator } from './workhub-coordination-coordinator.js';
201
202
  import { WorkHubActionEffectFailure, workHubResumedTurnId, } from './workhub-coordination-action-gate.js';
@@ -408,6 +409,7 @@ export async function createExecutionRuntimeHostComposition(context, options = {
408
409
  sessionQuery: pluginSessionQuery,
409
410
  sources: pluginSources,
410
411
  resolveTools: (sessionId, core) => pluginTools.resolve(sessionId, core),
412
+ readSessionUsage: (sessionId, from, to) => readSessionUsageWindow(openedUsageStores, sessionId, from, to),
411
413
  }));
412
414
  const isAssistantSession = (header) => header.labels?.some((label) => label === 'personal-assistant' || label === ASSISTANT_BACKGROUND_SESSION_LABEL) ?? false;
413
415
  const assertAssistantSessionEnabled = async (sessionId) => {
@@ -750,6 +750,11 @@ export async function resolveExecutionTarget(header, runtimePolicy, oauthCredent
750
750
  ? {}
751
751
  : { defaultApiProtocol: resolved.connection.defaultApiProtocol }),
752
752
  defaultModel: model,
753
+ ...(resolved.connection.providerType === 'openai-codex' &&
754
+ resolved.connection.modelSource === 'fetched' &&
755
+ resolved.connection.modelsFetchedAt !== undefined
756
+ ? { codexCompactionMetadataFetchedAt: resolved.connection.modelsFetchedAt }
757
+ : {}),
753
758
  models: discovered
754
759
  ? [...resolved.connection.models]
755
760
  : [...resolved.connection.models, modelInfo],
@@ -108,6 +108,8 @@ async function buildHostAiSdkBackend(input, target) {
108
108
  connectionId: input.context.header.llmConnectionId,
109
109
  providerStateIdentity: target.providerStateIdentity,
110
110
  modelId: target.model,
111
+ models: target.connection.models,
112
+ codexCompactionMetadataFetchedAt: target.connection.codexCompactionMetadataFetchedAt,
111
113
  providerOptions,
112
114
  }), textHistorySummarizer)
113
115
  : textHistorySummarizer;
@@ -26,7 +26,7 @@
26
26
  "@maka/core": "0.1.0",
27
27
  "@maka/runtime": "0.1.0",
28
28
  "@maka/storage": "0.1.0",
29
- "systeminformation": "^5.33.13",
29
+ "systeminformation": "^5.33.18",
30
30
  "ws": "^8.21.3",
31
31
  "yaml": "^2.9.1",
32
32
  "zod": "^4.6.5"
@@ -39,7 +39,7 @@ indent_style = off
39
39
  indent = off
40
40
  max_line_length = off
41
41
 
42
- [.nycrc]
42
+ [{.nycrc,*.mjs}]
43
43
  indent_style = tab
44
44
 
45
45
  [tea.yaml]
@@ -24,7 +24,7 @@ Caller Application → qs.stringify(obj, options) → Stringifying Engine → Ou
24
24
  ```
25
25
 
26
26
  **Trust Boundaries:**
27
- - **Input string (parse):** May come from untrusted sources (e.g., user input, network requests)
27
+ - **Input string (parse):** May come from untrusted sources (e.g., user input, network requests). `qs` defines its own key grammar, where bracket groups denote nesting, so the parameter names it derives are deliberately not the flat names produced by `URLSearchParams`, `node:querystring`, or a WHATWG-conformant parser. The parsed object, not the raw string, is the only thing a caller should make trust decisions about; see "Interpretation conflict" below.
28
28
  - **Input object (stringify):** The calling application's own data, not a direct untrusted channel. May contain cycles (handled by the cycle detector). Deeply-nested non-cyclic input surfaces as a catchable synchronous `RangeError` (stack overflow), not unbounded resource consumption; callers that serialize externally-influenced objects should bound their depth/size beforehand.
29
29
  - **Options:** Provided by the caller
30
30
  - **Cycle Tracking:** Used only during stringification to detect and handle circular references
@@ -51,6 +51,7 @@ Caller Application → qs.stringify(obj, options) → Stringifying Engine → Ou
51
51
  - **Information Disclosure:** Error messages may expose internal details or sensitive data.
52
52
  - **Denial of Service:** Large or malformed input can exhaust memory or CPU.
53
53
  - **Elevation of Privilege:** Prototype pollution can lead to unintended privilege escalation in the host application[3][4][8].
54
+ - **Interpretation conflict (out of scope):** An application that parses the same query string with two different parsers, and makes a trust decision on one result while consuming the other, can be attacked by any input the two parsers read differently. `qs`'s bracket grammar guarantees such inputs exist, so this is a property of the deployment, not a defect `qs` can remove. It is the calling application's responsibility to parse once and validate the parsed object.
54
55
 
55
56
  ### 6. Mitigation/Countermeasures
56
57
 
@@ -58,8 +59,9 @@ Caller Application → qs.stringify(obj, options) → Stringifying Engine → Ou
58
59
  |---------------------------------------------------|---------------------|
59
60
  | Tampering (malicious input, prototype pollution) | Strict input validation; keep `allowPrototypes: false` by default; use `plainObjects` for output; ensure builtins/globals are never modified by parse[4][8]. |
60
61
  | Information Disclosure (error messages) | Generic error messages without stack traces or internal paths. |
61
- | Denial of Service (memory/CPU exhaustion) | `arrayLimit`, `parameterLimit`, and depth limiting apply with safe defaults. Note that `arrayLimit` is a representation threshold (array vs. object), and `parameterLimit` bounds only the number of `&`-delimited parameters - neither caps the elements produced by `comma: true` splitting. For a hard cap that rejects oversized untrusted input, opt in to `throwOnLimitExceeded: true` (default `false`), and always bound input size at the transport layer[7]. On the `stringify` side, the input is the caller's own object rather than an untrusted string, so unbounded nesting manifests as a catchable `RangeError` (stack overflow) rather than exhaustion; callers serializing externally-influenced objects should bound depth/size beforehand, and may opt into `stringify`'s `depth` option (default `Infinity`) for a deterministic, catchable error. |
62
+ | Denial of Service (memory/CPU exhaustion) | `arrayLimit`, `parameterLimit`, and depth limiting apply with safe defaults. Note that `arrayLimit` is a representation threshold (array vs. object), and `parameterLimit` bounds only the number of `&`-delimited parameters - neither caps the elements produced by `comma: true` splitting. For hard caps on the number of parameters and on each array's length, opt in to `throwOnLimitExceeded: true` (default `false`), which rejects a query string with more than `parameterLimit` parameters, or one that would produce any array longer than `arrayLimit`. `arrayLimit` applies to each array separately, including each parameter's comma-split value, so the total number of values can still reach `parameterLimit` times `arrayLimit`; always bound input size at the transport layer as well[7]. On the `stringify` side, the input is the caller's own object rather than an untrusted string, so unbounded nesting manifests as a catchable `RangeError` (stack overflow) rather than exhaustion; callers serializing externally-influenced objects should bound depth/size beforehand, and may opt into `stringify`'s `depth` option (default `Infinity`) for a deterministic, catchable error. |
62
63
  | Elevation of Privilege (prototype pollution) | Keep `allowPrototypes: false`; validate options against allowlist; use `plainObjects` to avoid prototype pollution[4][8]. |
64
+ | Interpretation conflict (validate-then-use across two parsers) | None available inside `qs`, by construction: `qs` and a flat parser disagree about parameter names for well-formed input, so no hardening can make the two views equal. Parse untrusted input once, with `qs`, and apply deny lists, authorization, and signature checks to the parsed object rather than to raw key text. Text that `qs` cannot place in its grammar is currently dropped, so distinct raw names can share one parsed path ([#619](https://github.com/ljharb/qs/issues/619)). |
63
65
 
64
66
  ### 7. Risk Ranking
65
67
 
@@ -1,17 +1,64 @@
1
+ ## **6.17.0**
2
+ - [New] `stringify`: add `WHATWG` format (#623)
3
+ - [Fix] `stringify`: do not sort the caller's `filter` array in place (#637)
4
+ - [Fix] `parse`: copy an own `__proto__` key as data (#636)
5
+ - [Fix] `parse`: decode percent-encoded dots case-insensitively (#635)
6
+ - [Fix] `stringify`: encode keys when `allowEmptyArrays` is set (#561)
7
+ - [Fix] `parse`: preserve comma arrays when interpreting numeric entities (#630)
8
+ - [Fix] `parse`: allow scalar fields with a zero array limit (#627)
9
+ - [Fix] `parse`: do not count an existing comma array toward `arrayLimit` when `duplicates` discards or replaces it
10
+ - [Fix] `utils`: `encode`: encode lone surrogates as U+FFFD instead of consuming the next code unit (#624)
11
+ - [eslint] fix indentation
12
+ - [readme] document the encoder's `format` argument
13
+ - [meta] add changelog acknowledgement
14
+ - [meta] fix changelog indents
15
+ - [Dev Deps] update `eslint`, `evalmd`, `npmignore`
16
+ - [Tests] `stringify`: pin `allowEmptyArrays` output for keys that need no encoding
17
+ - [Tests] `stringify`: pin `sort` order for nested keys selected by an array `filter`
18
+ - [Tests] `parse`: cover `interpretNumericEntities` across value shapes, keys, and the charset sentinel
19
+ - [Tests] `parse`: cover more input that builds no array at a zero or negative `arrayLimit`
20
+ - [Tests] `parse`: cover `comma` with a zero or negative `arrayLimit`, and combining a full comma array
21
+ - [Tests] `parse`: pin the behavior of a zero or negative `arrayLimit`
22
+ - [Tests] `utils`/`stringify`: pin `~`, `*`, and parens encoding in RFC3986 and RFC1738
23
+
24
+ ## **6.16.1**
25
+ - [Fix] `parse`: do not mutate an object input
26
+ - [Fix] `stringify`: stop re-encoding structural separator dots when `encodeDotInKeys` is set (#564)
27
+ - [Fix] `stringify`: `arrayFormat: 'comma'`: pass the full encoder arguments with `encodeValuesOnly`
28
+ - [Fix] `utils`: encode `+` as `%2B` in iso-8859-1 mode so values round-trip (#565)
29
+ - [Fix] parse: only convert top-level dots to brackets when allowDots is on (#621)
30
+ - [Fix] `stringify`: ignore inherited array format names (#622)
31
+ - [Fix] `stringify`: `arrayFormat: 'comma'`: do not throw on elements with no callable `toString`
32
+ - [Fix] `utils`: `merge`: seed an overflow container's index from its own keys, not just the source
33
+ - [Fix] `utils`: `merge`: keep a `strictMerge` conflict container's index and `arrayLimit` correct
34
+ - [Fix] `parse`: only treat `[]` as array syntax when it ends the key
35
+ - [Perf] `decode`: skip decoders when there is no percent sign (#583)
36
+ - [Robustness] `utils`: `merge`: apply `arrayLimit` to the two-element array built for colliding values
37
+ - [Robustness] `parse`: bound the nesting added by repeated `strictMerge` conflicts (credit @Yeaseen)
38
+ - [Docs] document that `qs` and flat parsers disagree about parameter names
39
+ - [Docs] clarify that `throwOnLimitExceeded` limits each array, not the total number of values
40
+ - [Docs] clarify 6.14.1/6.15.2 parse bracket behavior change (#567, #568)
41
+ - [meta] fix changelog indents
42
+ - [actions] set least-privilege `cache-mode`
43
+ - [Tests] `utils`: `decode`: add missing cases
44
+ - [Tests] `parse`: characterize the `]=` split when an `=` precedes any bracket group (#620)
45
+ - [Tests] `parse`: `allowDots`: cover stray close brackets before a dot
46
+ - [Dev Deps] update `@ljharb/eslint-config`, `eslint`, `tape`
47
+
1
48
  ## **6.16.0**
2
- - [New] `stringify`: add a `depth` option to bound recursion depth (default `Infinity`)
3
- - [Fix] stringify: serialize Date values when a filter is provided
4
- - [Fix] `parse`: enforce `arrayLimit` on comma groups under `[]=` when `throwOnLimitExceeded` is set
5
- - [Fix] `parse`: flatten a collection appended to an overflowed array (#571)
6
- - [Fix] `utils`: `isBuffer`: do not invoke a non-callable `constructor.isBuffer`
7
- - [Fix] `stringify`: do not let `allowEmptyArrays` skip cycle detection (or drop own keys) on an empty array with own properties
8
- - [Fix] `stringify`: encode dots in a top-level key with a primitive value when encodeDotInKeys is set (#562)
9
- - [Docs] threat model: clarify `stringify` deep-nesting DoS is caller-bounded
10
- - [Docs] clarify `arrayLimit` is a representation threshold, not an element-count cap
11
- - [Tests] `parse`: remove a test that pinned `[]=` comma groups escaping `arrayLimit`
12
- - [Tests] `stringify`: pin current `encodeDotInKeys` separator-dot behavior
13
- - [Dev Deps] update `@ljharb/eslint-config`, `eslint`
14
- - [Dev Deps] update `eslint`, `evalmd`
49
+ - [New] `stringify`: add a `depth` option to bound recursion depth (default `Infinity`)
50
+ - [Fix] stringify: serialize Date values when a filter is provided
51
+ - [Fix] `parse`: enforce `arrayLimit` on comma groups under `[]=` when `throwOnLimitExceeded` is set
52
+ - [Fix] `parse`: flatten a collection appended to an overflowed array (#571)
53
+ - [Fix] `utils`: `isBuffer`: do not invoke a non-callable `constructor.isBuffer`
54
+ - [Fix] `stringify`: do not let `allowEmptyArrays` skip cycle detection (or drop own keys) on an empty array with own properties
55
+ - [Fix] `stringify`: encode dots in a top-level key with a primitive value when encodeDotInKeys is set (#562)
56
+ - [Docs] threat model: clarify `stringify` deep-nesting DoS is caller-bounded
57
+ - [Docs] clarify `arrayLimit` is a representation threshold, not an element-count cap
58
+ - [Tests] `parse`: remove a test that pinned `[]=` comma groups escaping `arrayLimit`
59
+ - [Tests] `stringify`: pin current `encodeDotInKeys` separator-dot behavior
60
+ - [Dev Deps] update `@ljharb/eslint-config`, `eslint`
61
+ - [Dev Deps] update `eslint`, `evalmd`
15
62
 
16
63
  ## **6.15.3**
17
64
  - [Fix] `parse`: enforce `throwOnLimitExceeded` for cumulative array growth via `combine`/`merge`
@@ -29,7 +76,7 @@
29
76
  - [Fix] `stringify`: use configured `delimiter` after `charsetSentinel` (#555)
30
77
  - [Fix] `stringify`: apply `formatter` to encoded key under `strictNullHandling` (#554)
31
78
  - [Fix] `stringify`: skip null/undefined filter-array entries instead of crashing in `encoder` (#551)
32
- - [Fix] `parse`: handle nested bracket groups and add regression tests (#530)
79
+ - [Fix] `parse`: handle nested bracket groups and add regression tests (#530); changes output for some unbalanced bracket keys (see #558)
33
80
  - [readme] fix grammar (#550)
34
81
  - [Dev Deps] update `@ljharb/eslint-config`
35
82
  - [Tests] add regression tests for keys containing percent-encoded bracket text
@@ -85,6 +85,23 @@ assert.deepEqual(qs.parse('foo[bar][baz]=foobarbaz'), {
85
85
  });
86
86
  ```
87
87
 
88
+ Since v6.14.1 and v6.15.2, `parse` interprets nested bracket groups more consistently. Query strings with unbalanced `[`/`]` can produce different results than in 6.15.1; see [#558](https://github.com/ljharb/qs/issues/558).
89
+
90
+ Because of that bracket grammar,
91
+ **qs** does not agree with `URLSearchParams`,
92
+ `node:querystring`, or any other flat parser about what the "name" of a parameter is,
93
+ and it never has.
94
+ `URLSearchParams` reads `'foo[bar]=baz'` as one flat name `foo[bar]`;
95
+ **qs** reads it as the path `foo.bar`.
96
+ Text that **qs**'s grammar cannot place,
97
+ such as the `extra` in `'foo[bar]extra=baz'`,
98
+ is currently dropped, so several distinct flat names can resolve to the same **qs** path
99
+ (see [#619](https://github.com/ljharb/qs/issues/619)).
100
+
101
+ So validating, authorizing, filtering,
102
+ or signing a query string with one parser and then consuming it with another is unsafe: a deny list or signature computed over `URLSearchParams` names does not constrain what **qs** will produce from the same string.
103
+ Parse the input once, with **qs**, and apply your checks to the resulting object.
104
+
88
105
  By default, when nesting objects **qs** will only parse up to 5 children deep.
89
106
  This means if you attempt to parse a string like `'a[b][c][d][e][f][g][h][i]=j'` your resulting object will be:
90
107
 
@@ -135,7 +152,7 @@ var limited = qs.parse('a=b&c=d', { parameterLimit: 1 });
135
152
  assert.deepEqual(limited, { a: 'b' });
136
153
  ```
137
154
 
138
- If you want an error to be thrown whenever the a limit is exceeded (eg, `parameterLimit`, `arrayLimit`), set the `throwOnLimitExceeded` option to `true`. This option will generate a descriptive error if the query string exceeds a configured limit.
155
+ If you want an error to be thrown whenever a limit is exceeded (eg, `parameterLimit`, `arrayLimit`), set the `throwOnLimitExceeded` option to `true`. This option will generate a descriptive error if the query string exceeds a configured limit.
139
156
  ```javascript
140
157
  try {
141
158
  qs.parse('a=1&b=2&c=3&d=4', { parameterLimit: 3, throwOnLimitExceeded: true });
@@ -147,7 +164,7 @@ try {
147
164
 
148
165
  When `throwOnLimitExceeded` is set to `false` (default), **qs** will parse up to the specified `parameterLimit` and ignore the rest without throwing an error.
149
166
 
150
- Note that `parameterLimit` only bounds the number of `&`-delimited parameters; it does not limit how many values a single parameter expands into. In particular, when `comma: true` is enabled, a single parameter's value is split on commas into arbitrarily many elements, which `parameterLimit` does not constrain. To bound the total element count from untrusted input, set `throwOnLimitExceeded: true` (see `arrayLimit` below), and always bound the input size at the transport layer (e.g. an HTTP body-size limit).
167
+ Note that `parameterLimit` only bounds the number of `&`-delimited parameters; it does not limit how many values a single parameter expands into. In particular, when `comma: true` is enabled, a single parameter's value is split on commas into arbitrarily many elements, which `parameterLimit` does not constrain. Setting `throwOnLimitExceeded: true` (see `arrayLimit` below) makes **qs** reject a query string with more than `parameterLimit` parameters, one that uses an array index of `arrayLimit` or greater, or one that would produce any array longer than `arrayLimit`, including an array produced by comma splitting. `arrayLimit` applies to each array separately, including each parameter's comma-split value, so the total number of values across all parameters can still reach `parameterLimit` times `arrayLimit`, which is 20,000 at the default limits; always bound the input size at the transport layer as well (e.g. an HTTP body-size limit).
151
168
 
152
169
  To bypass the leading question mark, use `ignoreQueryPrefix`:
153
170
 
@@ -293,6 +310,8 @@ assert.deepEqual(withIndexedEmptyString, { a: ['b', '', 'c'] });
293
310
  Any array members with an index of `20` or greater will instead be converted to an object with the index as the key.
294
311
  This is needed to handle cases when someone sent, for example, `a[999999999]` and it will take significant time to iterate over this huge array.
295
312
 
313
+ Since v6.14.1 and v6.15.2, the arrayLimit now applies to arrays specified using both the `[]` and `[0]` syntaxes.
314
+
296
315
  ```javascript
297
316
  var withMaxIndex = qs.parse('a[100]=b');
298
317
  assert.deepEqual(withMaxIndex, { a: { '100': 'b' } });
@@ -317,7 +336,7 @@ try {
317
336
 
318
337
  When `throwOnLimitExceeded` is set to `false` (default), **qs** will parse up to the specified `arrayLimit` and if the limit is exceeded, the array will instead be converted to an object with the index as the key.
319
338
 
320
- Note that `arrayLimit` is a *representation* threshold that controls when a numerically-indexed collection switches from an array to an object — it is **not** a hard cap on the total number of elements parsed. With the default `throwOnLimitExceeded: false`, exceeding `arrayLimit` never rejects or truncates input; it only changes the container type, and the resulting object still holds every element (so its size stays proportional to the input). This conversion is itself a safeguard: it avoids allocating a huge sparse array for input like `a[999999999]`. If you need a hard limit that rejects oversized input from untrusted sources, set `throwOnLimitExceeded: true`.
339
+ Note that `arrayLimit` is a *representation* threshold that controls when a numerically-indexed collection switches from an array to an object; it is **not** a hard cap on the total number of elements parsed. With the default `throwOnLimitExceeded: false`, exceeding `arrayLimit` never rejects or truncates input; it only changes the container type, and the resulting object still holds every element (so its size stays proportional to the input). This conversion is itself a safeguard: it avoids allocating a huge sparse array for input like `a[999999999]`. If you need a hard limit that rejects oversized arrays in untrusted query strings, set `throwOnLimitExceeded: true`. `throwOnLimitExceeded` still limits each array separately, not the total: with `comma: true`, each comma-separated group under `a[]=` counts as one element of the outer array, and can itself hold up to `arrayLimit` elements.
321
340
 
322
341
  To prevent array syntax (`a[]`, `a[0]`) from being parsed as arrays, set `parseArrays` to `false`.
323
342
  Note that duplicate keys (e.g. `a=b&a=c`) may still produce arrays when `duplicates` is `'combine'` (the default).
@@ -361,6 +380,15 @@ assert.deepEqual(arraysOfObjects, { a: ['b', 'c'] })
361
380
  ```
362
381
  (_this cannot convert nested objects, such as `a={b:1},{c:d}`_)
363
382
 
383
+ With `[]` or index notation, the values from each comma-separated group stay together as one element of the array, while repeated plain keys flatten into one array:
384
+ ```javascript
385
+ var nestedGroups = qs.parse('a[]=b,c&a[]=d', { comma: true });
386
+ assert.deepEqual(nestedGroups, { a: [['b', 'c'], 'd'] });
387
+
388
+ var flatGroups = qs.parse('a=b,c&a=d', { comma: true });
389
+ assert.deepEqual(flatGroups, { a: ['b', 'c', 'd'] });
390
+ ```
391
+
364
392
  ### Parsing primitive/scalar values (numbers, booleans, null, etc)
365
393
 
366
394
  By default, all values are parsed as strings.
@@ -449,6 +477,20 @@ var encoded = qs.stringify({ a: { b: 'c' } }, { encoder: function (str, defaultE
449
477
  }})
450
478
  ```
451
479
 
480
+ The encoder also receives the `format` option as a fifth argument.
481
+ An encoder that wraps `defaultEncoder` should pass all of its arguments through,
482
+ since the default encoding depends on `format` (for example, RFC1738 leaves parentheses unencoded):
483
+
484
+ ```javascript
485
+ var wrapped = qs.stringify({ a: 'b(c)' }, {
486
+ format: 'RFC1738',
487
+ encoder: function (str, defaultEncoder, charset, type, format) {
488
+ return defaultEncoder(str, defaultEncoder, charset, type, format);
489
+ }
490
+ });
491
+ assert.equal(wrapped, 'a=b(c)');
492
+ ```
493
+
452
494
  The type argument is also provided to the decoder:
453
495
 
454
496
  ```javascript
@@ -730,7 +772,8 @@ var obj = qs.parse('a=%82%B1%82%F1%82%C9%82%BF%82%CD%81I', { decoder: decoder })
730
772
  assert.deepEqual(obj, { a: 'こんにちは!' });
731
773
  ```
732
774
 
733
- ### RFC 3986 and RFC 1738 space encoding
775
+ <a name="rfc-3986-and-rfc-1738-space-encoding"></a>
776
+ ### Encoding formats: RFC 3986, RFC 1738, and WHATWG
734
777
 
735
778
  RFC3986 used as default option and encodes ' ' to *%20* which is backward compatible.
736
779
  In the same time, output can be stringified as per RFC1738 with ' ' equal to '+'.
@@ -741,6 +784,16 @@ assert.equal(qs.stringify({ a: 'b c' }, { format : 'RFC3986' }), 'a=b%20c');
741
784
  assert.equal(qs.stringify({ a: 'b c' }, { format : 'RFC1738' }), 'a=b+c');
742
785
  ```
743
786
 
787
+ To percent-encode keys and values the same way as the [WHATWG `application/x-www-form-urlencoded` serializer](https://url.spec.whatwg.org/#concept-urlencoded-serializer) (what `URLSearchParams` and HTML form submission use), use the `WHATWG` format.
788
+ Like RFC1738 it encodes ' ' as '+'; it also leaves `*` unencoded, but percent-encodes `~`, `(`, and `)`.
789
+ It only affects encoding; how `null`, dates, arrays, and objects are serialized is unchanged:
790
+
791
+ ```javascript
792
+ assert.equal(qs.stringify({ a: 'b c~*()' }, { format: 'WHATWG' }), 'a=b+c%7E*%28%29');
793
+ ```
794
+
795
+ _(Note: the `WHATWG` format matches `URLSearchParams` encoding only with the default `utf-8` charset, and only when keys are encoded too, so not with `encodeValuesOnly`. With `charset: 'iso-8859-1'`, every format uses the same `escape`-based encoding, which leaves characters like `@` and `/` unencoded; only the space handling differs)_
796
+
744
797
  ## Security
745
798
 
746
799
  Please email [@ljharb](https://github.com/ljharb) or see https://tidelift.com/security if you have a potential security vulnerability to report.