@senad-d/observme 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +4 -0
- package/CHANGELOG.md +48 -0
- package/README.md +36 -11
- package/dashboards/observme-agent-node-graphs.json +5 -5
- package/dashboards/observme-agents.json +169 -4
- package/dashboards/observme-alerts.yaml +16 -3
- package/dashboards/observme-overview.json +6 -6
- package/dashboards/observme-trace-journey.json +4 -4
- package/docs/agent-subagent-observability-requirements.md +19 -10
- package/docs/compatibility-matrix.md +21 -4
- package/docs/configuration.md +7 -2
- package/docs/extension-integration.md +20 -13
- package/docs/reference/03-pi-event-and-session-model.md +6 -6
- package/docs/reference/04-telemetry-semantic-conventions.md +39 -1
- package/docs/reference/05-otel-pipeline-and-collector.md +23 -10
- package/docs/reference/06-security-privacy-redaction.md +13 -6
- package/docs/reference/08-query-grafana-integration.md +30 -7
- package/docs/reference/09-dashboards-alerts-slos.md +132 -3
- package/docs/reference/10-testing-release-operations.md +44 -14
- package/docs/reference/11-deployment-runbooks.md +82 -5
- package/docs/reference/12-configuration-reference.md +48 -5
- package/docs/review-validation.md +17 -0
- package/examples/README.md +7 -2
- package/examples/collector.yaml +8 -8
- package/examples/integrations/subagent-runner.ts +8 -5
- package/examples/observme.yaml +3 -2
- package/package.json +14 -4
- package/src/commands/obs-agents.ts +17 -12
- package/src/commands/obs-backfill.ts +2 -2
- package/src/commands/obs-command-support.ts +2 -1
- package/src/commands/obs-cost.ts +15 -7
- package/src/commands/obs-health.ts +22 -2
- package/src/commands/obs-loki-summary.ts +4 -8
- package/src/commands/obs-session.ts +35 -28
- package/src/commands/obs-status.ts +56 -13
- package/src/commands/obs-tools.ts +19 -8
- package/src/commands/obs-trace.ts +9 -0
- package/src/commands/obs.ts +6 -1
- package/src/config/bootstrap-project-config.ts +50 -32
- package/src/config/defaults.ts +1 -2
- package/src/config/load-config.ts +270 -81
- package/src/config/project-paths.ts +318 -6
- package/src/config/query-limits.ts +10 -0
- package/src/config/schema.ts +18 -8
- package/src/config/transport-security.ts +107 -0
- package/src/config/validate.ts +88 -12
- package/src/extension.ts +2 -12
- package/src/integration.ts +6 -2
- package/src/otel/logs.ts +24 -13
- package/src/otel/metrics.ts +24 -13
- package/src/otel/otlp-endpoint.ts +41 -2
- package/src/otel/otlp-http-options.ts +11 -0
- package/src/otel/sdk.ts +155 -9
- package/src/otel/shutdown.ts +39 -11
- package/src/otel/traces.ts +34 -16
- package/src/pi/active-agent-lease.ts +101 -0
- package/src/pi/agent-tree-tracker.ts +14 -1
- package/src/pi/compatibility.ts +121 -0
- package/src/pi/event-handlers/agent-turn.ts +16 -9
- package/src/pi/event-handlers/lifecycle.ts +312 -106
- package/src/pi/event-handlers/llm.ts +17 -9
- package/src/pi/event-handlers/session-events.ts +53 -19
- package/src/pi/event-handlers/tool-bash.ts +21 -27
- package/src/pi/handler-internals.ts +16 -26
- package/src/pi/handler-runtime.ts +159 -54
- package/src/pi/handler-types.ts +40 -17
- package/src/pi/handlers.ts +12 -1
- package/src/pi/integration-api.ts +27 -15
- package/src/pi/session-correlation.ts +167 -0
- package/src/pi/subagent-spawn.ts +164 -31
- package/src/privacy/redact.ts +574 -68
- package/src/privacy/secret-patterns.ts +6 -1
- package/src/query/grafana-readiness.ts +18 -2
- package/src/query/grafana-transport.ts +150 -27
- package/src/query/grafana-url.ts +36 -0
- package/src/query/grafana.ts +4 -136
- package/src/query/loki.ts +2 -4
- package/src/query/prometheus.ts +8 -10
- package/src/query/tempo.ts +2 -4
- package/src/query/trace-link.ts +186 -0
- package/src/safety/display-bounds.ts +84 -0
- package/src/semconv/metrics.ts +7 -0
- package/src/semconv/values.ts +2 -0
package/src/privacy/redact.ts
CHANGED
|
@@ -89,22 +89,82 @@ export interface CustomRedactionPatternSafetyIssue {
|
|
|
89
89
|
| "custom_redaction_pattern_too_long"
|
|
90
90
|
| "custom_redaction_pattern_unsupported_construct"
|
|
91
91
|
| "custom_redaction_pattern_nested_quantifier"
|
|
92
|
+
| "custom_redaction_pattern_ambiguous_alternation"
|
|
93
|
+
| "custom_redaction_pattern_ambiguous_repetition"
|
|
92
94
|
| "custom_redaction_pattern_empty_match"
|
|
93
95
|
| "invalid_custom_redaction_pattern";
|
|
94
96
|
readonly message: string;
|
|
95
97
|
}
|
|
96
98
|
|
|
97
|
-
interface
|
|
98
|
-
|
|
99
|
+
interface RegexCharacterRange {
|
|
100
|
+
readonly start: number;
|
|
101
|
+
readonly end: number;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
type RegexCharacterDomain = readonly RegexCharacterRange[];
|
|
105
|
+
|
|
106
|
+
type RegexSafetyNode =
|
|
107
|
+
| RegexEmptyNode
|
|
108
|
+
| RegexCharacterNode
|
|
109
|
+
| RegexSequenceNode
|
|
110
|
+
| RegexAlternationNode
|
|
111
|
+
| RegexRepetitionNode;
|
|
112
|
+
|
|
113
|
+
interface RegexEmptyNode {
|
|
114
|
+
readonly kind: "empty";
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
interface RegexCharacterNode {
|
|
118
|
+
readonly kind: "character";
|
|
119
|
+
readonly domain: RegexCharacterDomain | undefined;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
interface RegexSequenceNode {
|
|
123
|
+
readonly kind: "sequence";
|
|
124
|
+
readonly children: readonly RegexSafetyNode[];
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
interface RegexAlternationNode {
|
|
128
|
+
readonly kind: "alternation";
|
|
129
|
+
readonly branches: readonly RegexSafetyNode[];
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
interface RegexRepetitionNode {
|
|
133
|
+
readonly kind: "repetition";
|
|
134
|
+
readonly child: RegexSafetyNode;
|
|
135
|
+
readonly minimum: number;
|
|
136
|
+
readonly maximum: number;
|
|
137
|
+
readonly variable: boolean;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
interface RegexSafetyParserState {
|
|
141
|
+
readonly source: string;
|
|
142
|
+
readonly caseInsensitive: boolean;
|
|
143
|
+
index: number;
|
|
144
|
+
failed: boolean;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
interface RegexQuantifier {
|
|
148
|
+
readonly minimum: number;
|
|
149
|
+
readonly maximum: number;
|
|
150
|
+
readonly end: number;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
interface RegexEscapeToken {
|
|
154
|
+
readonly end: number;
|
|
155
|
+
readonly assertion: boolean;
|
|
156
|
+
readonly domain: RegexCharacterDomain | undefined;
|
|
157
|
+
readonly singleton: number | undefined;
|
|
99
158
|
}
|
|
100
159
|
|
|
101
|
-
interface
|
|
102
|
-
readonly
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
lastClosedGroupHadQuantifier: boolean;
|
|
160
|
+
interface RegexCharacterClassToken {
|
|
161
|
+
readonly end: number;
|
|
162
|
+
readonly domain: RegexCharacterDomain | undefined;
|
|
163
|
+
readonly singleton: number | undefined;
|
|
106
164
|
}
|
|
107
165
|
|
|
166
|
+
type UnsafeQuantifiedGroupKind = "nested_quantifier" | "ambiguous_alternation" | "ambiguous_repetition";
|
|
167
|
+
|
|
108
168
|
export function redactValue(rawValue: string, options: RedactionOptions): RedactionResult {
|
|
109
169
|
const stages: RedactionStage[] = [];
|
|
110
170
|
try {
|
|
@@ -396,7 +456,7 @@ export function validateCustomRedactionPattern(pattern: string): CustomRedaction
|
|
|
396
456
|
if (pattern.length > MAX_CUSTOM_REDACTION_PATTERN_CHARS) return [createCustomRedactionPatternTooLongIssue()];
|
|
397
457
|
|
|
398
458
|
const source = normalizeCustomRedactionPatternSource(pattern);
|
|
399
|
-
const unsafeIssue = detectUnsafeCustomRedactionPattern(source);
|
|
459
|
+
const unsafeIssue = detectUnsafeCustomRedactionPattern(source, pattern.startsWith("(?i)"));
|
|
400
460
|
if (unsafeIssue) return [unsafeIssue];
|
|
401
461
|
|
|
402
462
|
return validateCustomRedactionPatternSyntax(source);
|
|
@@ -407,9 +467,16 @@ export function normalizeCustomRedactionPatternSource(pattern: string): string {
|
|
|
407
467
|
return pattern;
|
|
408
468
|
}
|
|
409
469
|
|
|
410
|
-
export function detectUnsafeCustomRedactionPattern(
|
|
470
|
+
export function detectUnsafeCustomRedactionPattern(
|
|
471
|
+
source: string,
|
|
472
|
+
caseInsensitive = false,
|
|
473
|
+
): CustomRedactionPatternSafetyIssue | undefined {
|
|
411
474
|
if (hasUnsupportedCustomRedactionConstruct(source)) return createCustomRedactionPatternUnsupportedConstructIssue();
|
|
412
|
-
|
|
475
|
+
|
|
476
|
+
const unsafeQuantifiedGroup = detectUnsafeQuantifiedGroup(source, caseInsensitive);
|
|
477
|
+
if (unsafeQuantifiedGroup === "nested_quantifier") return createCustomRedactionPatternNestedQuantifierIssue();
|
|
478
|
+
if (unsafeQuantifiedGroup === "ambiguous_alternation") return createCustomRedactionPatternAmbiguousAlternationIssue();
|
|
479
|
+
if (unsafeQuantifiedGroup === "ambiguous_repetition") return createCustomRedactionPatternAmbiguousRepetitionIssue();
|
|
413
480
|
return undefined;
|
|
414
481
|
}
|
|
415
482
|
|
|
@@ -436,68 +503,511 @@ export function hasUnsupportedCustomRedactionConstruct(source: string): boolean
|
|
|
436
503
|
return false;
|
|
437
504
|
}
|
|
438
505
|
|
|
439
|
-
export function hasNestedQuantifiedGroup(source: string): boolean {
|
|
440
|
-
const
|
|
506
|
+
export function hasNestedQuantifiedGroup(source: string, caseInsensitive = false): boolean {
|
|
507
|
+
const tree = parseRegexSafetyPattern(source, caseInsensitive);
|
|
508
|
+
return tree ? hasNestedRegexRepetition(tree) : false;
|
|
509
|
+
}
|
|
441
510
|
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
511
|
+
export function hasQuantifiedAlternationGroup(source: string, caseInsensitive = false): boolean {
|
|
512
|
+
const tree = parseRegexSafetyPattern(source, caseInsensitive);
|
|
513
|
+
return tree ? hasAmbiguousRepeatedAlternation(tree) : false;
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
export function detectUnsafeQuantifiedGroup(
|
|
517
|
+
source: string,
|
|
518
|
+
caseInsensitive = false,
|
|
519
|
+
): UnsafeQuantifiedGroupKind | undefined {
|
|
520
|
+
const tree = parseRegexSafetyPattern(source, caseInsensitive);
|
|
521
|
+
if (!tree) return undefined;
|
|
522
|
+
if (hasNestedRegexRepetition(tree)) return "nested_quantifier";
|
|
523
|
+
if (hasAmbiguousRepeatedAlternation(tree)) return "ambiguous_alternation";
|
|
524
|
+
if (hasAmbiguousSequentialRepetition(tree)) return "ambiguous_repetition";
|
|
525
|
+
return undefined;
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
function parseRegexSafetyPattern(source: string, caseInsensitive: boolean): RegexSafetyNode | undefined {
|
|
529
|
+
const state: RegexSafetyParserState = { source, caseInsensitive, index: 0, failed: false };
|
|
530
|
+
const tree = parseRegexSafetyAlternation(state);
|
|
531
|
+
if (state.failed || state.index !== source.length) return undefined;
|
|
532
|
+
return tree;
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
function parseRegexSafetyAlternation(state: RegexSafetyParserState): RegexSafetyNode {
|
|
536
|
+
const branches: RegexSafetyNode[] = [parseRegexSafetySequence(state)];
|
|
537
|
+
while (!state.failed && state.source[state.index] === "|") {
|
|
538
|
+
state.index += 1;
|
|
539
|
+
branches.push(parseRegexSafetySequence(state));
|
|
449
540
|
}
|
|
541
|
+
if (branches.length === 1) return branches[0];
|
|
542
|
+
return { kind: "alternation", branches };
|
|
543
|
+
}
|
|
450
544
|
|
|
451
|
-
|
|
545
|
+
function parseRegexSafetySequence(state: RegexSafetyParserState): RegexSafetyNode {
|
|
546
|
+
const children: RegexSafetyNode[] = [];
|
|
547
|
+
while (!state.failed && state.index < state.source.length) {
|
|
548
|
+
const char = state.source[state.index];
|
|
549
|
+
if (char === "|" || char === ")") break;
|
|
550
|
+
children.push(parseRegexSafetyQuantifiedAtom(state));
|
|
551
|
+
}
|
|
552
|
+
if (children.length === 0) return { kind: "empty" };
|
|
553
|
+
if (children.length === 1) return children[0];
|
|
554
|
+
return { kind: "sequence", children };
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
function parseRegexSafetyQuantifiedAtom(state: RegexSafetyParserState): RegexSafetyNode {
|
|
558
|
+
const atom = parseRegexSafetyAtom(state);
|
|
559
|
+
if (state.failed) return atom;
|
|
560
|
+
|
|
561
|
+
const quantifier = readRegexQuantifier(state.source, state.index);
|
|
562
|
+
if (!quantifier) return atom;
|
|
563
|
+
state.index = quantifier.end;
|
|
564
|
+
if (state.source[state.index] === "?") state.index += 1;
|
|
565
|
+
if (quantifier.minimum === 1 && quantifier.maximum === 1) return atom;
|
|
566
|
+
if (quantifier.maximum === 0) return { kind: "empty" };
|
|
567
|
+
return {
|
|
568
|
+
kind: "repetition",
|
|
569
|
+
child: atom,
|
|
570
|
+
minimum: quantifier.minimum,
|
|
571
|
+
maximum: quantifier.maximum,
|
|
572
|
+
variable: quantifier.minimum !== quantifier.maximum,
|
|
573
|
+
};
|
|
452
574
|
}
|
|
453
575
|
|
|
454
|
-
function
|
|
455
|
-
|
|
576
|
+
function parseRegexSafetyAtom(state: RegexSafetyParserState): RegexSafetyNode {
|
|
577
|
+
const char = state.source[state.index];
|
|
578
|
+
if (char === "(") return parseRegexSafetyGroup(state);
|
|
579
|
+
if (char === "[") return parseRegexSafetyCharacterClass(state);
|
|
580
|
+
if (char === "\\") return parseRegexSafetyEscape(state);
|
|
581
|
+
if (char === ".") {
|
|
582
|
+
state.index += 1;
|
|
583
|
+
return { kind: "character", domain: undefined };
|
|
584
|
+
}
|
|
585
|
+
if (char === "^" || char === "$") {
|
|
586
|
+
state.index += 1;
|
|
587
|
+
return { kind: "empty" };
|
|
588
|
+
}
|
|
589
|
+
if (char === undefined || "*+?{})".includes(char)) {
|
|
590
|
+
state.failed = true;
|
|
591
|
+
return { kind: "empty" };
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
const codePoint = state.source.codePointAt(state.index);
|
|
595
|
+
if (codePoint === undefined) {
|
|
596
|
+
state.failed = true;
|
|
597
|
+
return { kind: "empty" };
|
|
598
|
+
}
|
|
599
|
+
state.index += codePoint > 0xffff ? 2 : 1;
|
|
600
|
+
return { kind: "character", domain: createRegexCharacterDomain(codePoint, state.caseInsensitive) };
|
|
456
601
|
}
|
|
457
602
|
|
|
458
|
-
function
|
|
459
|
-
if (
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
603
|
+
function parseRegexSafetyGroup(state: RegexSafetyParserState): RegexSafetyNode {
|
|
604
|
+
if (state.source.startsWith("(?:", state.index)) state.index += 3;
|
|
605
|
+
else state.index += 1;
|
|
606
|
+
|
|
607
|
+
const child = parseRegexSafetyAlternation(state);
|
|
608
|
+
if (state.source[state.index] !== ")") {
|
|
609
|
+
state.failed = true;
|
|
610
|
+
return child;
|
|
463
611
|
}
|
|
612
|
+
state.index += 1;
|
|
613
|
+
return child;
|
|
614
|
+
}
|
|
464
615
|
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
616
|
+
function parseRegexSafetyCharacterClass(state: RegexSafetyParserState): RegexSafetyNode {
|
|
617
|
+
const closeIndex = findRegexCharacterClassEnd(state.source, state.index + 1);
|
|
618
|
+
if (closeIndex === -1) {
|
|
619
|
+
state.failed = true;
|
|
620
|
+
return { kind: "empty" };
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
const domain = readRegexCharacterClassDomain(
|
|
624
|
+
state.source,
|
|
625
|
+
state.index + 1,
|
|
626
|
+
closeIndex,
|
|
627
|
+
state.caseInsensitive,
|
|
628
|
+
);
|
|
629
|
+
state.index = closeIndex + 1;
|
|
630
|
+
return { kind: "character", domain };
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
function parseRegexSafetyEscape(state: RegexSafetyParserState): RegexSafetyNode {
|
|
634
|
+
const token = readRegexEscapeToken(state.source, state.index, false, state.caseInsensitive);
|
|
635
|
+
state.index = token.end;
|
|
636
|
+
if (token.assertion) return { kind: "empty" };
|
|
637
|
+
return { kind: "character", domain: token.domain };
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
function readRegexQuantifier(source: string, index: number): RegexQuantifier | undefined {
|
|
641
|
+
const char = source[index];
|
|
642
|
+
if (char === "*") return { minimum: 0, maximum: Number.POSITIVE_INFINITY, end: index + 1 };
|
|
643
|
+
if (char === "+") return { minimum: 1, maximum: Number.POSITIVE_INFINITY, end: index + 1 };
|
|
644
|
+
if (char === "?") return { minimum: 0, maximum: 1, end: index + 1 };
|
|
645
|
+
if (char !== "{") return undefined;
|
|
646
|
+
|
|
647
|
+
const closeIndex = source.indexOf("}", index + 1);
|
|
648
|
+
if (closeIndex === -1) return undefined;
|
|
649
|
+
const body = source.slice(index + 1, closeIndex);
|
|
650
|
+
if (!/^\d+(?:,\d*)?$/u.test(body)) return undefined;
|
|
651
|
+
|
|
652
|
+
const commaIndex = body.indexOf(",");
|
|
653
|
+
if (commaIndex === -1) {
|
|
654
|
+
const exact = Number(body);
|
|
655
|
+
return { minimum: exact, maximum: exact, end: closeIndex + 1 };
|
|
656
|
+
}
|
|
657
|
+
const minimum = Number(body.slice(0, commaIndex));
|
|
658
|
+
const maximumSource = body.slice(commaIndex + 1);
|
|
659
|
+
const maximum = maximumSource.length === 0 ? Number.POSITIVE_INFINITY : Number(maximumSource);
|
|
660
|
+
return { minimum, maximum, end: closeIndex + 1 };
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
function findRegexCharacterClassEnd(source: string, start: number): number {
|
|
664
|
+
let escaped = false;
|
|
665
|
+
for (let index = start; index < source.length; index += 1) {
|
|
666
|
+
if (escaped) {
|
|
667
|
+
escaped = false;
|
|
668
|
+
continue;
|
|
669
|
+
}
|
|
670
|
+
if (source[index] === "\\") {
|
|
671
|
+
escaped = true;
|
|
672
|
+
continue;
|
|
673
|
+
}
|
|
674
|
+
if (source[index] === "]") return index;
|
|
675
|
+
}
|
|
676
|
+
return -1;
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
function readRegexCharacterClassDomain(
|
|
680
|
+
source: string,
|
|
681
|
+
start: number,
|
|
682
|
+
end: number,
|
|
683
|
+
caseInsensitive: boolean,
|
|
684
|
+
): RegexCharacterDomain | undefined {
|
|
685
|
+
if (source[start] === "^") return undefined;
|
|
686
|
+
|
|
687
|
+
const ranges: RegexCharacterRange[] = [];
|
|
688
|
+
let index = start;
|
|
689
|
+
while (index < end) {
|
|
690
|
+
const left = readRegexCharacterClassToken(source, index);
|
|
691
|
+
if (!left.domain) return undefined;
|
|
692
|
+
index = left.end;
|
|
693
|
+
if (source[index] === "-" && index + 1 < end) {
|
|
694
|
+
const right = readRegexCharacterClassToken(source, index + 1);
|
|
695
|
+
if (left.singleton === undefined || right.singleton === undefined || left.singleton > right.singleton) {
|
|
696
|
+
return undefined;
|
|
697
|
+
}
|
|
698
|
+
ranges.push({ start: left.singleton, end: right.singleton });
|
|
699
|
+
index = right.end;
|
|
700
|
+
continue;
|
|
701
|
+
}
|
|
702
|
+
ranges.push(...left.domain);
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
const domain = mergeRegexCharacterRanges(ranges);
|
|
706
|
+
return caseInsensitive ? addAsciiCaseVariants(domain) : domain;
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
function readRegexCharacterClassToken(source: string, index: number): RegexCharacterClassToken {
|
|
710
|
+
if (source[index] === "\\") {
|
|
711
|
+
const token = readRegexEscapeToken(source, index, true, false);
|
|
712
|
+
return { end: token.end, domain: token.domain, singleton: token.singleton };
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
const codePoint = source.codePointAt(index);
|
|
716
|
+
if (codePoint === undefined) return { end: index + 1, domain: undefined, singleton: undefined };
|
|
717
|
+
const end = index + (codePoint > 0xffff ? 2 : 1);
|
|
718
|
+
return { end, domain: [{ start: codePoint, end: codePoint }], singleton: codePoint };
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
function readRegexEscapeToken(
|
|
722
|
+
source: string,
|
|
723
|
+
index: number,
|
|
724
|
+
inCharacterClass: boolean,
|
|
725
|
+
caseInsensitive: boolean,
|
|
726
|
+
): RegexEscapeToken {
|
|
727
|
+
const escaped = source[index + 1];
|
|
728
|
+
const end = findRegexEscapeEnd(source, index, escaped);
|
|
729
|
+
if (!inCharacterClass && (escaped === "b" || escaped === "B")) {
|
|
730
|
+
return { end, assertion: true, domain: [], singleton: undefined };
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
const shorthandDomain = regexShorthandDomain(escaped);
|
|
734
|
+
if (shorthandDomain) return { end, assertion: false, domain: shorthandDomain, singleton: undefined };
|
|
735
|
+
const singleton = decodeRegexEscapeCodePoint(source, index, escaped, inCharacterClass);
|
|
736
|
+
const domain = singleton === undefined ? undefined : createRegexCharacterDomain(singleton, caseInsensitive);
|
|
737
|
+
return { end, assertion: false, domain, singleton };
|
|
469
738
|
}
|
|
470
739
|
|
|
471
|
-
function
|
|
472
|
-
if (
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
return
|
|
740
|
+
function findRegexEscapeEnd(source: string, index: number, escaped: string | undefined): number {
|
|
741
|
+
if (escaped === undefined) return source.length;
|
|
742
|
+
if ((escaped === "p" || escaped === "P") && source[index + 2] === "{") {
|
|
743
|
+
const closeIndex = source.indexOf("}", index + 3);
|
|
744
|
+
return closeIndex === -1 ? source.length : closeIndex + 1;
|
|
476
745
|
}
|
|
746
|
+
if (escaped === "u" && source[index + 2] === "{") {
|
|
747
|
+
const closeIndex = source.indexOf("}", index + 3);
|
|
748
|
+
return closeIndex === -1 ? source.length : closeIndex + 1;
|
|
749
|
+
}
|
|
750
|
+
if (escaped === "u") return Math.min(source.length, index + 6);
|
|
751
|
+
if (escaped === "x") return Math.min(source.length, index + 4);
|
|
752
|
+
if (escaped === "c") return Math.min(source.length, index + 3);
|
|
753
|
+
return Math.min(source.length, index + 2);
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
function regexShorthandDomain(escaped: string | undefined): RegexCharacterDomain | undefined {
|
|
757
|
+
if (escaped === "d") return [{ start: 0x30, end: 0x39 }];
|
|
758
|
+
if (escaped === "w") {
|
|
759
|
+
return [
|
|
760
|
+
{ start: 0x30, end: 0x39 },
|
|
761
|
+
{ start: 0x41, end: 0x5a },
|
|
762
|
+
{ start: 0x5f, end: 0x5f },
|
|
763
|
+
{ start: 0x61, end: 0x7a },
|
|
764
|
+
];
|
|
765
|
+
}
|
|
766
|
+
if (escaped === "s") {
|
|
767
|
+
return [
|
|
768
|
+
{ start: 0x09, end: 0x0d },
|
|
769
|
+
{ start: 0x20, end: 0x20 },
|
|
770
|
+
{ start: 0xa0, end: 0xa0 },
|
|
771
|
+
{ start: 0x1680, end: 0x1680 },
|
|
772
|
+
{ start: 0x2000, end: 0x200a },
|
|
773
|
+
{ start: 0x2028, end: 0x2029 },
|
|
774
|
+
{ start: 0x202f, end: 0x202f },
|
|
775
|
+
{ start: 0x205f, end: 0x205f },
|
|
776
|
+
{ start: 0x3000, end: 0x3000 },
|
|
777
|
+
{ start: 0xfeff, end: 0xfeff },
|
|
778
|
+
];
|
|
779
|
+
}
|
|
780
|
+
return undefined;
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
function decodeRegexEscapeCodePoint(
|
|
784
|
+
source: string,
|
|
785
|
+
index: number,
|
|
786
|
+
escaped: string | undefined,
|
|
787
|
+
inCharacterClass: boolean,
|
|
788
|
+
): number | undefined {
|
|
789
|
+
if (escaped === undefined) return undefined;
|
|
790
|
+
const simpleEscapes: Readonly<Record<string, number>> = {
|
|
791
|
+
"0": 0x00,
|
|
792
|
+
b: inCharacterClass ? 0x08 : 0x62,
|
|
793
|
+
f: 0x0c,
|
|
794
|
+
n: 0x0a,
|
|
795
|
+
r: 0x0d,
|
|
796
|
+
t: 0x09,
|
|
797
|
+
v: 0x0b,
|
|
798
|
+
};
|
|
799
|
+
if (simpleEscapes[escaped] !== undefined) return simpleEscapes[escaped];
|
|
800
|
+
if (escaped === "x") return parseRegexHexEscape(source.slice(index + 2, index + 4));
|
|
801
|
+
if (escaped === "u" && source[index + 2] === "{") {
|
|
802
|
+
const closeIndex = source.indexOf("}", index + 3);
|
|
803
|
+
return closeIndex === -1 ? undefined : parseRegexHexEscape(source.slice(index + 3, closeIndex));
|
|
804
|
+
}
|
|
805
|
+
if (escaped === "u") return parseRegexHexEscape(source.slice(index + 2, index + 6));
|
|
806
|
+
if (escaped === "c") {
|
|
807
|
+
const controlCodePoint = source.codePointAt(index + 2);
|
|
808
|
+
return controlCodePoint === undefined ? undefined : controlCodePoint % 32;
|
|
809
|
+
}
|
|
810
|
+
if ("dDsSwWpPkK123456789".includes(escaped)) return undefined;
|
|
811
|
+
return escaped.codePointAt(0);
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
function parseRegexHexEscape(source: string): number | undefined {
|
|
815
|
+
if (!/^[\da-f]+$/iu.test(source)) return undefined;
|
|
816
|
+
const codePoint = Number.parseInt(source, 16);
|
|
817
|
+
return Number.isFinite(codePoint) ? codePoint : undefined;
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
function createRegexCharacterDomain(codePoint: number, caseInsensitive: boolean): RegexCharacterDomain | undefined {
|
|
821
|
+
const domain = [{ start: codePoint, end: codePoint }];
|
|
822
|
+
return caseInsensitive ? addAsciiCaseVariants(domain) : domain;
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
function addAsciiCaseVariants(domain: RegexCharacterDomain): RegexCharacterDomain | undefined {
|
|
826
|
+
const ranges: RegexCharacterRange[] = [...domain];
|
|
827
|
+
for (const range of domain) {
|
|
828
|
+
if (range.end > 0x7f) return undefined;
|
|
829
|
+
const upperStart = Math.max(range.start, 0x41);
|
|
830
|
+
const upperEnd = Math.min(range.end, 0x5a);
|
|
831
|
+
if (upperStart <= upperEnd) ranges.push({ start: upperStart + 0x20, end: upperEnd + 0x20 });
|
|
832
|
+
const lowerStart = Math.max(range.start, 0x61);
|
|
833
|
+
const lowerEnd = Math.min(range.end, 0x7a);
|
|
834
|
+
if (lowerStart <= lowerEnd) ranges.push({ start: lowerStart - 0x20, end: lowerEnd - 0x20 });
|
|
835
|
+
}
|
|
836
|
+
return mergeRegexCharacterRanges(ranges);
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
function mergeRegexCharacterRanges(ranges: readonly RegexCharacterRange[]): RegexCharacterDomain {
|
|
840
|
+
const sorted = [...ranges].sort(compareRegexCharacterRanges);
|
|
841
|
+
const merged: RegexCharacterRange[] = [];
|
|
842
|
+
for (const range of sorted) {
|
|
843
|
+
const previous = merged.at(-1);
|
|
844
|
+
if (!previous || range.start > previous.end + 1) {
|
|
845
|
+
merged.push(range);
|
|
846
|
+
continue;
|
|
847
|
+
}
|
|
848
|
+
if (range.end > previous.end) merged[merged.length - 1] = { start: previous.start, end: range.end };
|
|
849
|
+
}
|
|
850
|
+
return merged;
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
function compareRegexCharacterRanges(left: RegexCharacterRange, right: RegexCharacterRange): number {
|
|
854
|
+
if (left.start !== right.start) return left.start - right.start;
|
|
855
|
+
return left.end - right.end;
|
|
856
|
+
}
|
|
477
857
|
|
|
478
|
-
|
|
479
|
-
|
|
858
|
+
function regexNodeCanMatchEmpty(node: RegexSafetyNode): boolean {
|
|
859
|
+
if (node.kind === "empty") return true;
|
|
860
|
+
if (node.kind === "character") return false;
|
|
861
|
+
if (node.kind === "repetition") return node.minimum === 0 || regexNodeCanMatchEmpty(node.child);
|
|
862
|
+
if (node.kind === "alternation") {
|
|
863
|
+
for (const branch of node.branches) if (regexNodeCanMatchEmpty(branch)) return true;
|
|
864
|
+
return false;
|
|
865
|
+
}
|
|
866
|
+
for (const child of node.children) if (!regexNodeCanMatchEmpty(child)) return false;
|
|
480
867
|
return true;
|
|
481
868
|
}
|
|
482
869
|
|
|
483
|
-
function
|
|
484
|
-
if (
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
870
|
+
function regexNodeFirstDomain(node: RegexSafetyNode): RegexCharacterDomain | undefined {
|
|
871
|
+
if (node.kind === "empty") return [];
|
|
872
|
+
if (node.kind === "character") return node.domain;
|
|
873
|
+
if (node.kind === "repetition") return regexNodeFirstDomain(node.child);
|
|
874
|
+
if (node.kind === "alternation") return unionRegexNodeDomains(node.branches, false);
|
|
875
|
+
return unionRegexNodeDomains(node.children, true);
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
function unionRegexNodeDomains(
|
|
879
|
+
nodes: readonly RegexSafetyNode[],
|
|
880
|
+
stopAfterRequiredNode: boolean,
|
|
881
|
+
): RegexCharacterDomain | undefined {
|
|
882
|
+
const ranges: RegexCharacterRange[] = [];
|
|
883
|
+
for (const node of nodes) {
|
|
884
|
+
const domain = regexNodeFirstDomain(node);
|
|
885
|
+
if (!domain) return undefined;
|
|
886
|
+
ranges.push(...domain);
|
|
887
|
+
if (stopAfterRequiredNode && !regexNodeCanMatchEmpty(node)) break;
|
|
488
888
|
}
|
|
889
|
+
return mergeRegexCharacterRanges(ranges);
|
|
890
|
+
}
|
|
489
891
|
|
|
490
|
-
|
|
491
|
-
|
|
892
|
+
function regexDomainsAreProvablyDisjoint(
|
|
893
|
+
left: RegexCharacterDomain | undefined,
|
|
894
|
+
right: RegexCharacterDomain | undefined,
|
|
895
|
+
): boolean {
|
|
896
|
+
if (!left || !right) return false;
|
|
897
|
+
for (const leftRange of left) {
|
|
898
|
+
for (const rightRange of right) {
|
|
899
|
+
if (leftRange.start <= rightRange.end && rightRange.start <= leftRange.end) return false;
|
|
900
|
+
}
|
|
901
|
+
}
|
|
492
902
|
return true;
|
|
493
903
|
}
|
|
494
904
|
|
|
495
|
-
function
|
|
496
|
-
if (
|
|
497
|
-
if (
|
|
905
|
+
function regexNodeHasRepetition(node: RegexSafetyNode): boolean {
|
|
906
|
+
if (node.kind === "repetition") return true;
|
|
907
|
+
if (node.kind === "sequence") {
|
|
908
|
+
for (const child of node.children) if (regexNodeHasRepetition(child)) return true;
|
|
909
|
+
}
|
|
910
|
+
if (node.kind === "alternation") {
|
|
911
|
+
for (const branch of node.branches) if (regexNodeHasRepetition(branch)) return true;
|
|
912
|
+
}
|
|
913
|
+
return false;
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
function hasNestedRegexRepetition(node: RegexSafetyNode): boolean {
|
|
917
|
+
if (node.kind === "repetition") {
|
|
918
|
+
if (regexNodeHasRepetition(node.child)) return true;
|
|
919
|
+
return hasNestedRegexRepetition(node.child);
|
|
920
|
+
}
|
|
921
|
+
if (node.kind === "sequence") {
|
|
922
|
+
for (const child of node.children) if (hasNestedRegexRepetition(child)) return true;
|
|
923
|
+
}
|
|
924
|
+
if (node.kind === "alternation") {
|
|
925
|
+
for (const branch of node.branches) if (hasNestedRegexRepetition(branch)) return true;
|
|
926
|
+
}
|
|
927
|
+
return false;
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
function hasAmbiguousRepeatedAlternation(node: RegexSafetyNode): boolean {
|
|
931
|
+
if (node.kind === "repetition") {
|
|
932
|
+
if (hasAmbiguousAlternationWithin(node.child)) return true;
|
|
933
|
+
return hasAmbiguousRepeatedAlternation(node.child);
|
|
934
|
+
}
|
|
935
|
+
if (node.kind === "sequence") {
|
|
936
|
+
for (const child of node.children) if (hasAmbiguousRepeatedAlternation(child)) return true;
|
|
937
|
+
}
|
|
938
|
+
if (node.kind === "alternation") {
|
|
939
|
+
for (const branch of node.branches) if (hasAmbiguousRepeatedAlternation(branch)) return true;
|
|
940
|
+
}
|
|
941
|
+
return false;
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
function hasAmbiguousAlternationWithin(node: RegexSafetyNode): boolean {
|
|
945
|
+
if (node.kind === "alternation") {
|
|
946
|
+
if (regexAlternationBranchesOverlap(node)) return true;
|
|
947
|
+
for (const branch of node.branches) if (hasAmbiguousAlternationWithin(branch)) return true;
|
|
948
|
+
}
|
|
949
|
+
if (node.kind === "sequence") {
|
|
950
|
+
for (const child of node.children) if (hasAmbiguousAlternationWithin(child)) return true;
|
|
951
|
+
}
|
|
952
|
+
if (node.kind === "repetition") return hasAmbiguousAlternationWithin(node.child);
|
|
953
|
+
return false;
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
function regexAlternationBranchesOverlap(node: RegexAlternationNode): boolean {
|
|
957
|
+
for (const branch of node.branches) if (regexNodeCanMatchEmpty(branch)) return true;
|
|
958
|
+
for (let leftIndex = 0; leftIndex < node.branches.length; leftIndex += 1) {
|
|
959
|
+
const leftDomain = regexNodeFirstDomain(node.branches[leftIndex]);
|
|
960
|
+
for (let rightIndex = leftIndex + 1; rightIndex < node.branches.length; rightIndex += 1) {
|
|
961
|
+
const rightDomain = regexNodeFirstDomain(node.branches[rightIndex]);
|
|
962
|
+
if (!regexDomainsAreProvablyDisjoint(leftDomain, rightDomain)) return true;
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
return false;
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
function hasAmbiguousSequentialRepetition(node: RegexSafetyNode): boolean {
|
|
969
|
+
if (node.kind === "sequence") return regexSequenceHasAmbiguousRepetition(node);
|
|
970
|
+
if (node.kind === "alternation") {
|
|
971
|
+
for (const branch of node.branches) if (hasAmbiguousSequentialRepetition(branch)) return true;
|
|
972
|
+
}
|
|
973
|
+
if (node.kind === "repetition") return hasAmbiguousSequentialRepetition(node.child);
|
|
974
|
+
return false;
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
function regexSequenceHasAmbiguousRepetition(node: RegexSequenceNode): boolean {
|
|
978
|
+
const previousRepetitions: RegexRepetitionNode[] = [];
|
|
979
|
+
for (const child of node.children) {
|
|
980
|
+
if (hasAmbiguousSequentialRepetition(child)) return true;
|
|
981
|
+
const currentRepetitions = collectVariableRegexRepetitions(child);
|
|
982
|
+
if (regexRepetitionCollectionsOverlap(previousRepetitions, currentRepetitions)) return true;
|
|
983
|
+
previousRepetitions.push(...currentRepetitions);
|
|
984
|
+
}
|
|
985
|
+
return false;
|
|
986
|
+
}
|
|
498
987
|
|
|
499
|
-
|
|
500
|
-
|
|
988
|
+
function collectVariableRegexRepetitions(node: RegexSafetyNode): RegexRepetitionNode[] {
|
|
989
|
+
if (node.kind === "repetition") return node.variable ? [node] : [];
|
|
990
|
+
const repetitions: RegexRepetitionNode[] = [];
|
|
991
|
+
if (node.kind === "sequence") {
|
|
992
|
+
for (const child of node.children) repetitions.push(...collectVariableRegexRepetitions(child));
|
|
993
|
+
}
|
|
994
|
+
if (node.kind === "alternation") {
|
|
995
|
+
for (const branch of node.branches) repetitions.push(...collectVariableRegexRepetitions(branch));
|
|
996
|
+
}
|
|
997
|
+
return repetitions;
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
function regexRepetitionCollectionsOverlap(
|
|
1001
|
+
left: readonly RegexRepetitionNode[],
|
|
1002
|
+
right: readonly RegexRepetitionNode[],
|
|
1003
|
+
): boolean {
|
|
1004
|
+
for (const leftRepetition of left) {
|
|
1005
|
+
const leftDomain = regexNodeFirstDomain(leftRepetition.child);
|
|
1006
|
+
for (const rightRepetition of right) {
|
|
1007
|
+
const rightDomain = regexNodeFirstDomain(rightRepetition.child);
|
|
1008
|
+
if (!regexDomainsAreProvablyDisjoint(leftDomain, rightDomain)) return true;
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
501
1011
|
return false;
|
|
502
1012
|
}
|
|
503
1013
|
|
|
@@ -516,24 +1026,6 @@ export function isBackreferenceEscape(source: string, index: number): boolean {
|
|
|
516
1026
|
return /[1-9]/u.test(char) || (char === "k" && source[index + 1] === "<");
|
|
517
1027
|
}
|
|
518
1028
|
|
|
519
|
-
export function isRegexQuantifierStart(source: string, index: number): boolean {
|
|
520
|
-
const char = source[index];
|
|
521
|
-
if (char === "?" && source[index - 1] === "(") return false;
|
|
522
|
-
return char === "*" || char === "+" || char === "?" || isBraceQuantifierStart(source, index);
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
export function isBraceQuantifierStart(source: string, index: number): boolean {
|
|
526
|
-
if (source[index] !== "{") return false;
|
|
527
|
-
const closeIndex = source.indexOf("}", index + 1);
|
|
528
|
-
if (closeIndex === -1) return false;
|
|
529
|
-
return /^\d+(?:,\d*)?$/u.test(source.slice(index + 1, closeIndex));
|
|
530
|
-
}
|
|
531
|
-
|
|
532
|
-
export function markCurrentRegexGroupQuantified(groupStack: RegexGroupScanState[]): void {
|
|
533
|
-
const currentGroup = groupStack.at(-1);
|
|
534
|
-
if (currentGroup) currentGroup.hasQuantifier = true;
|
|
535
|
-
}
|
|
536
|
-
|
|
537
1029
|
export function addCustomRedactionPatternIndex(
|
|
538
1030
|
issue: CustomRedactionPatternSafetyIssue,
|
|
539
1031
|
index: number,
|
|
@@ -572,6 +1064,20 @@ export function createCustomRedactionPatternNestedQuantifierIssue(): CustomRedac
|
|
|
572
1064
|
};
|
|
573
1065
|
}
|
|
574
1066
|
|
|
1067
|
+
export function createCustomRedactionPatternAmbiguousAlternationIssue(): CustomRedactionPatternSafetyIssue {
|
|
1068
|
+
return {
|
|
1069
|
+
code: "custom_redaction_pattern_ambiguous_alternation",
|
|
1070
|
+
message: "Repeated custom redaction alternatives must begin with provably disjoint characters.",
|
|
1071
|
+
};
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
export function createCustomRedactionPatternAmbiguousRepetitionIssue(): CustomRedactionPatternSafetyIssue {
|
|
1075
|
+
return {
|
|
1076
|
+
code: "custom_redaction_pattern_ambiguous_repetition",
|
|
1077
|
+
message: "Custom redaction patterns must not combine repetitions with overlapping starting characters.",
|
|
1078
|
+
};
|
|
1079
|
+
}
|
|
1080
|
+
|
|
575
1081
|
export function createCustomRedactionPatternEmptyMatchIssue(): CustomRedactionPatternSafetyIssue {
|
|
576
1082
|
return {
|
|
577
1083
|
code: "custom_redaction_pattern_empty_match",
|