@withgateway/sdk 0.2.5 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +3121 -1283
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +3185 -1347
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +9 -0
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +9 -0
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +280 -8
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +272 -8
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-1xnxt0Cc.d.ts → index-BO2SNSgy.d.ts} +12 -1
- package/dist/{index-B0R3IgsV.d.mts → index-BaKu5ZSU.d.mts} +124 -2
- package/dist/{index-B0R3IgsV.d.ts → index-BaKu5ZSU.d.ts} +124 -2
- package/dist/{index-C1wDEwH4.d.mts → index-Cz8Q6Keo.d.mts} +12 -1
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +726 -167
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +718 -167
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +9 -0
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +9 -0
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +97 -61
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +97 -61
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +154 -4
- package/dist/worlds/index.d.ts +154 -4
- package/dist/worlds/index.js +348 -97
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +348 -97
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +1 -1
- package/runtime/gatewaysdk/environment/_materialize.py +53 -9
- package/runtime/gatewaysdk/environment/_toml.py +13 -0
- package/runtime/gatewaysdk/environment/schema/api.py +3 -7
- package/runtime/gatewaysdk/environment/schema/batch.py +15 -11
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +24 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +60 -28
- package/runtime/gatewaysdk/environment/schema/conform.py +1 -1
- package/runtime/gatewaysdk/environment/schema/connector.py +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.ingest.toml +25 -0
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +17 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +1 -5
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +1 -5
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/platform.py +16 -4
- package/runtime/gatewaysdk/environment/schema/redaction.py +1 -1
- package/runtime/gatewaysdk/environment/schema/scaffold.py +9 -2
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +67 -9
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +61 -2
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +11 -11
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
- package/runtime/gatewaysdk/environment/schema/snapshot.py +1 -5
- package/runtime/gatewaysdk/environment/schema/tools_world.py +1 -5
- package/runtime/gatewaysdk/environment/schema/ui.py +1 -5
- package/runtime/gatewaysdk/environment/schema/world_tests.py +1 -1
- package/runtime/gatewaysdk/environment/world.py +120 -4
- package/skills/benchmarks-getting-started/SKILL.md +67 -9
- package/skills/connector-schema-authoring/connector-worlds.md +8 -1
- package/skills/connector-schema-authoring/world-schema.md +1 -1
- package/skills/hooks-getting-started/SKILL.md +61 -2
- package/skills/world-data-ingestion/SKILL.md +4 -1
- package/skills/worlds-getting-started/SKILL.md +11 -11
- package/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
package/dist/hooks/index.mjs
CHANGED
|
@@ -1,6 +1,49 @@
|
|
|
1
1
|
import { createRequire as __gatewayCreateRequire } from "module";
|
|
2
2
|
const require = __gatewayCreateRequire(import.meta.url);
|
|
3
3
|
|
|
4
|
+
// src/hooks/caseSwap.ts
|
|
5
|
+
var CASE_SWAP_MIN_CHARS = 4;
|
|
6
|
+
var standingAlone = /* @__PURE__ */ new Map();
|
|
7
|
+
function standingAlonePattern(needle) {
|
|
8
|
+
let pattern = standingAlone.get(needle);
|
|
9
|
+
if (pattern === void 0) {
|
|
10
|
+
const escaped = needle.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
11
|
+
pattern = new RegExp(`(?<![A-Za-z0-9])${escaped}(?![A-Za-z0-9])`, "gi");
|
|
12
|
+
standingAlone.set(needle, pattern);
|
|
13
|
+
}
|
|
14
|
+
return pattern;
|
|
15
|
+
}
|
|
16
|
+
var WORD = /[A-Za-z0-9]+/g;
|
|
17
|
+
function shapedWord(word, seen) {
|
|
18
|
+
const hasLetter = seen.toLowerCase() !== seen.toUpperCase();
|
|
19
|
+
if (hasLetter && seen === seen.toUpperCase()) return word.toUpperCase();
|
|
20
|
+
if (hasLetter && seen === seen.toLowerCase()) return word.toLowerCase();
|
|
21
|
+
const first = seen.slice(0, 1);
|
|
22
|
+
if (first !== first.toLowerCase())
|
|
23
|
+
return word.slice(0, 1).toUpperCase() + word.slice(1);
|
|
24
|
+
return word;
|
|
25
|
+
}
|
|
26
|
+
function shaped(replacement, seen) {
|
|
27
|
+
const seenWords = seen.match(WORD) ?? [];
|
|
28
|
+
if (seenWords.length !== (replacement.match(WORD) ?? []).length)
|
|
29
|
+
return shapedWord(replacement, seen);
|
|
30
|
+
let index2 = 0;
|
|
31
|
+
return replacement.replace(
|
|
32
|
+
WORD,
|
|
33
|
+
(word) => shapedWord(word, seenWords[index2++])
|
|
34
|
+
);
|
|
35
|
+
}
|
|
36
|
+
function swapValue(text, needle, replacement) {
|
|
37
|
+
if (!needle) return text;
|
|
38
|
+
const out = text.split(needle).join(replacement);
|
|
39
|
+
if (needle.length < CASE_SWAP_MIN_CHARS) return out;
|
|
40
|
+
if (!out.toLowerCase().includes(needle.toLowerCase())) return out;
|
|
41
|
+
return out.replace(
|
|
42
|
+
standingAlonePattern(needle),
|
|
43
|
+
(seen) => shaped(replacement, seen)
|
|
44
|
+
);
|
|
45
|
+
}
|
|
46
|
+
|
|
4
47
|
// src/hooks/alias.ts
|
|
5
48
|
import { createHash, createHmac, randomBytes } from "crypto";
|
|
6
49
|
var GIVEN = [
|
|
@@ -835,10 +878,9 @@ var AliasMap = class _AliasMap {
|
|
|
835
878
|
const [inbound] = this.pairs();
|
|
836
879
|
let out = text;
|
|
837
880
|
for (const [needle, replacement, alias] of inbound) {
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
}
|
|
881
|
+
const swapped = swapValue(out, needle, replacement);
|
|
882
|
+
if (swapped !== out) alias.hitsIn += 1;
|
|
883
|
+
out = swapped;
|
|
842
884
|
}
|
|
843
885
|
out = out.replace(SHA256_ANY_GLOBAL, (match) => this.mintDigest(match));
|
|
844
886
|
if (this.mintInbound.length) out = this.mintFoundIn(out, where);
|
|
@@ -854,10 +896,9 @@ var AliasMap = class _AliasMap {
|
|
|
854
896
|
const [, outbound] = this.pairs();
|
|
855
897
|
let out = text;
|
|
856
898
|
for (const [needle, replacement, alias] of outbound) {
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
}
|
|
899
|
+
const swapped = swapValue(out, needle, replacement);
|
|
900
|
+
if (swapped !== out) alias.hitsOut += 1;
|
|
901
|
+
out = swapped;
|
|
861
902
|
}
|
|
862
903
|
const problems = [];
|
|
863
904
|
for (const [needle] of outbound) {
|
|
@@ -3493,6 +3534,221 @@ var LiveTools = class _LiveTools {
|
|
|
3493
3534
|
}
|
|
3494
3535
|
};
|
|
3495
3536
|
|
|
3537
|
+
// src/hooks/audit.ts
|
|
3538
|
+
var MIN_VALUE_CHARS = 4;
|
|
3539
|
+
var REVIEW_MIN_CHARS = 20;
|
|
3540
|
+
var ANY_TOOL = "*";
|
|
3541
|
+
var isObject4 = (node) => typeof node === "object" && node !== null && !Array.isArray(node);
|
|
3542
|
+
function* stringsOf(node) {
|
|
3543
|
+
if (typeof node === "string") {
|
|
3544
|
+
yield node;
|
|
3545
|
+
return;
|
|
3546
|
+
}
|
|
3547
|
+
const children = Array.isArray(node) ? node : isObject4(node) ? Object.values(node) : [];
|
|
3548
|
+
for (const child of children) yield* stringsOf(child);
|
|
3549
|
+
}
|
|
3550
|
+
function* fieldsOf(node) {
|
|
3551
|
+
if (Array.isArray(node)) {
|
|
3552
|
+
for (const child of node) yield* fieldsOf(child);
|
|
3553
|
+
return;
|
|
3554
|
+
}
|
|
3555
|
+
if (!isObject4(node)) return;
|
|
3556
|
+
for (const [key2, value] of Object.entries(node)) {
|
|
3557
|
+
yield [key2, value];
|
|
3558
|
+
yield* fieldsOf(value);
|
|
3559
|
+
}
|
|
3560
|
+
}
|
|
3561
|
+
var Identifier = class {
|
|
3562
|
+
constructor(kind, value, source) {
|
|
3563
|
+
this.kind = kind;
|
|
3564
|
+
this.value = value;
|
|
3565
|
+
this.source = source;
|
|
3566
|
+
}
|
|
3567
|
+
get key() {
|
|
3568
|
+
return folded(this.value);
|
|
3569
|
+
}
|
|
3570
|
+
get prefix() {
|
|
3571
|
+
return this.value.slice(0, 3);
|
|
3572
|
+
}
|
|
3573
|
+
};
|
|
3574
|
+
var IdentifierSource = class {
|
|
3575
|
+
ignored() {
|
|
3576
|
+
return [];
|
|
3577
|
+
}
|
|
3578
|
+
};
|
|
3579
|
+
var LegendSource = class extends IdentifierSource {
|
|
3580
|
+
name = "legend";
|
|
3581
|
+
entries;
|
|
3582
|
+
constructor(legend) {
|
|
3583
|
+
super();
|
|
3584
|
+
this.entries = legend.filter((entry) => entry.real);
|
|
3585
|
+
}
|
|
3586
|
+
collect() {
|
|
3587
|
+
return this.entries.filter((entry) => (entry.role ?? "private") !== "public").map((entry) => new Identifier(entry.kind ?? "reference", String(entry.real), this.name));
|
|
3588
|
+
}
|
|
3589
|
+
ignored() {
|
|
3590
|
+
const open = this.entries.filter((e) => e.role === "public").map((e) => String(e.real));
|
|
3591
|
+
const personas = this.entries.filter((e) => e.alias).map((e) => String(e.alias));
|
|
3592
|
+
return [...open, ...personas];
|
|
3593
|
+
}
|
|
3594
|
+
};
|
|
3595
|
+
var FieldSource = class extends IdentifierSource {
|
|
3596
|
+
constructor(fieldKinds) {
|
|
3597
|
+
super();
|
|
3598
|
+
this.fieldKinds = fieldKinds;
|
|
3599
|
+
}
|
|
3600
|
+
name = "fields";
|
|
3601
|
+
kindsFor(tool) {
|
|
3602
|
+
return { ...this.fieldKinds[ANY_TOOL] ?? {}, ...this.fieldKinds[tool] ?? {} };
|
|
3603
|
+
}
|
|
3604
|
+
*collect(raw) {
|
|
3605
|
+
for (const record of raw) {
|
|
3606
|
+
const kinds = this.kindsFor(String(record.tool ?? ""));
|
|
3607
|
+
for (const [key2, value] of fieldsOf(record.result)) {
|
|
3608
|
+
const kind = kinds[key2];
|
|
3609
|
+
if (kind === void 0) continue;
|
|
3610
|
+
for (const text of stringsOf(value)) yield new Identifier(kind, text, this.name);
|
|
3611
|
+
}
|
|
3612
|
+
}
|
|
3613
|
+
}
|
|
3614
|
+
};
|
|
3615
|
+
var FinderSource = class extends IdentifierSource {
|
|
3616
|
+
name;
|
|
3617
|
+
kinds;
|
|
3618
|
+
constructor(kinds = INBOUND_KINDS) {
|
|
3619
|
+
super();
|
|
3620
|
+
const bad = kinds.filter((kind) => !INBOUND_KINDS.includes(kind));
|
|
3621
|
+
if (bad.length > 0)
|
|
3622
|
+
throw new Error(
|
|
3623
|
+
`${bad.join(", ")} cannot be found by a pattern; only ${INBOUND_KINDS.join(", ")} can`
|
|
3624
|
+
);
|
|
3625
|
+
this.kinds = [...kinds];
|
|
3626
|
+
this.name = `finders:${this.kinds.join(",")}`;
|
|
3627
|
+
}
|
|
3628
|
+
*collect(raw) {
|
|
3629
|
+
for (const record of raw)
|
|
3630
|
+
for (const text of stringsOf(record.result))
|
|
3631
|
+
for (const kind of this.kinds)
|
|
3632
|
+
for (const value of findKind(kind, text)) yield new Identifier(kind, value, "finders");
|
|
3633
|
+
}
|
|
3634
|
+
};
|
|
3635
|
+
var IdentifierCensus = class {
|
|
3636
|
+
constructor(sources) {
|
|
3637
|
+
this.sources = sources;
|
|
3638
|
+
}
|
|
3639
|
+
get detectors() {
|
|
3640
|
+
return this.sources.map((source) => source.name);
|
|
3641
|
+
}
|
|
3642
|
+
build(raw) {
|
|
3643
|
+
const ignored = new Set(this.sources.flatMap((s) => [...s.ignored()].map(folded)));
|
|
3644
|
+
const seen = /* @__PURE__ */ new Map();
|
|
3645
|
+
for (const source of this.sources)
|
|
3646
|
+
for (const identifier of source.collect(raw)) {
|
|
3647
|
+
if (identifier.value.length < MIN_VALUE_CHARS || ignored.has(identifier.key)) continue;
|
|
3648
|
+
if (!seen.has(identifier.key)) seen.set(identifier.key, identifier);
|
|
3649
|
+
}
|
|
3650
|
+
return [...seen.values()];
|
|
3651
|
+
}
|
|
3652
|
+
};
|
|
3653
|
+
function shuffled(items, seed) {
|
|
3654
|
+
const out = [...items];
|
|
3655
|
+
let state = seed >>> 0;
|
|
3656
|
+
const next = () => {
|
|
3657
|
+
state = state + 1831565813 >>> 0;
|
|
3658
|
+
let t = Math.imul(state ^ state >>> 15, 1 | state);
|
|
3659
|
+
t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
|
|
3660
|
+
return ((t ^ t >>> 14) >>> 0) / 4294967296;
|
|
3661
|
+
};
|
|
3662
|
+
for (let i = out.length - 1; i > 0; i -= 1) {
|
|
3663
|
+
const j = Math.floor(next() * (i + 1));
|
|
3664
|
+
[out[i], out[j]] = [out[j], out[i]];
|
|
3665
|
+
}
|
|
3666
|
+
return out;
|
|
3667
|
+
}
|
|
3668
|
+
var AuditReport = class {
|
|
3669
|
+
constructor(findings, census, detectors, recordsScanned, leakedIdentifiers, cleanTexts) {
|
|
3670
|
+
this.findings = findings;
|
|
3671
|
+
this.census = census;
|
|
3672
|
+
this.detectors = detectors;
|
|
3673
|
+
this.recordsScanned = recordsScanned;
|
|
3674
|
+
this.leakedIdentifiers = leakedIdentifiers;
|
|
3675
|
+
this.cleanTexts = cleanTexts;
|
|
3676
|
+
}
|
|
3677
|
+
get ok() {
|
|
3678
|
+
return this.findings.length === 0;
|
|
3679
|
+
}
|
|
3680
|
+
get censusTotal() {
|
|
3681
|
+
return this.census.length;
|
|
3682
|
+
}
|
|
3683
|
+
/** The share of census identifiers that did NOT reach the model; 1 for an empty census. */
|
|
3684
|
+
get coverage() {
|
|
3685
|
+
if (this.census.length === 0) return 1;
|
|
3686
|
+
return Math.round((1 - this.leakedIdentifiers / this.census.length) * 1e4) / 1e4;
|
|
3687
|
+
}
|
|
3688
|
+
count(pick3) {
|
|
3689
|
+
const counts = {};
|
|
3690
|
+
for (const identifier of this.census) counts[pick3(identifier)] = (counts[pick3(identifier)] ?? 0) + 1;
|
|
3691
|
+
return counts;
|
|
3692
|
+
}
|
|
3693
|
+
/** `size` strings the model saw that nothing flagged, for a person to read. */
|
|
3694
|
+
reviewSample(size, seed = 0) {
|
|
3695
|
+
const pool = [...this.cleanTexts].sort(
|
|
3696
|
+
(a, b) => a.record - b.record || a.text.localeCompare(b.text)
|
|
3697
|
+
);
|
|
3698
|
+
return shuffled(pool, seed).slice(0, size);
|
|
3699
|
+
}
|
|
3700
|
+
toDict() {
|
|
3701
|
+
return {
|
|
3702
|
+
ok: this.ok,
|
|
3703
|
+
records_scanned: this.recordsScanned,
|
|
3704
|
+
detectors: this.detectors,
|
|
3705
|
+
census: {
|
|
3706
|
+
total: this.censusTotal,
|
|
3707
|
+
by_source: this.count((i) => i.source),
|
|
3708
|
+
by_kind: this.count((i) => i.kind)
|
|
3709
|
+
},
|
|
3710
|
+
leaked_identifiers: this.leakedIdentifiers,
|
|
3711
|
+
coverage: this.coverage,
|
|
3712
|
+
findings: this.findings
|
|
3713
|
+
};
|
|
3714
|
+
}
|
|
3715
|
+
};
|
|
3716
|
+
var RedactionAudit = class _RedactionAudit {
|
|
3717
|
+
constructor(census) {
|
|
3718
|
+
this.census = census;
|
|
3719
|
+
}
|
|
3720
|
+
/** The standard audit: the run's legend, declared field kinds, pattern finders. */
|
|
3721
|
+
static forRun(options) {
|
|
3722
|
+
return new _RedactionAudit(
|
|
3723
|
+
new IdentifierCensus([
|
|
3724
|
+
new LegendSource(options.legend ?? []),
|
|
3725
|
+
new FieldSource(options.fieldKinds ?? {}),
|
|
3726
|
+
new FinderSource(options.finderKinds ?? INBOUND_KINDS)
|
|
3727
|
+
])
|
|
3728
|
+
);
|
|
3729
|
+
}
|
|
3730
|
+
run(raw, shown) {
|
|
3731
|
+
const census = this.census.build(raw);
|
|
3732
|
+
const findings = [];
|
|
3733
|
+
const clean = [];
|
|
3734
|
+
const leaked = /* @__PURE__ */ new Set();
|
|
3735
|
+
shown.forEach((record, index2) => {
|
|
3736
|
+
const tool = String(record.tool ?? "");
|
|
3737
|
+
const texts = [...stringsOf(record.result)];
|
|
3738
|
+
const haystack = folded(texts.join("\n"));
|
|
3739
|
+
const hits = census.filter((identifier) => haystack.includes(identifier.key));
|
|
3740
|
+
hits.forEach((identifier) => leaked.add(identifier.key));
|
|
3741
|
+
findings.push(
|
|
3742
|
+
...hits.map((i) => ({ record: index2, tool, kind: i.kind, source: i.source, prefix: i.prefix }))
|
|
3743
|
+
);
|
|
3744
|
+
clean.push(
|
|
3745
|
+
...texts.filter((t) => t.length >= REVIEW_MIN_CHARS && !hits.some((i) => folded(t).includes(i.key))).map((text) => ({ record: index2, tool, text }))
|
|
3746
|
+
);
|
|
3747
|
+
});
|
|
3748
|
+
return new AuditReport(findings, census, this.census.detectors, shown.length, leaked.size, clean);
|
|
3749
|
+
}
|
|
3750
|
+
};
|
|
3751
|
+
|
|
3496
3752
|
// src/hooks/run.ts
|
|
3497
3753
|
import { randomBytes as randomBytes2 } from "crypto";
|
|
3498
3754
|
import * as fs4 from "fs";
|
|
@@ -4162,6 +4418,7 @@ var HookedSession = class {
|
|
|
4162
4418
|
export {
|
|
4163
4419
|
AliasHandler,
|
|
4164
4420
|
AliasMap,
|
|
4421
|
+
AuditReport,
|
|
4165
4422
|
CONTEXT_KEY,
|
|
4166
4423
|
CallableTool,
|
|
4167
4424
|
CommandHandler,
|
|
@@ -4169,6 +4426,8 @@ export {
|
|
|
4169
4426
|
DenyHandler,
|
|
4170
4427
|
EVENTS,
|
|
4171
4428
|
EVENT_ALIASES,
|
|
4429
|
+
FieldSource,
|
|
4430
|
+
FinderSource,
|
|
4172
4431
|
FunctionHandler,
|
|
4173
4432
|
HANDLER_TYPES,
|
|
4174
4433
|
HookConfigError,
|
|
@@ -4180,13 +4439,18 @@ export {
|
|
|
4180
4439
|
HttpTool,
|
|
4181
4440
|
INBOUND_KINDS,
|
|
4182
4441
|
INPUT_TYPES,
|
|
4442
|
+
Identifier,
|
|
4443
|
+
IdentifierCensus,
|
|
4444
|
+
IdentifierSource,
|
|
4183
4445
|
KINDS,
|
|
4446
|
+
LegendSource,
|
|
4184
4447
|
LiveToolError,
|
|
4185
4448
|
LiveTools,
|
|
4186
4449
|
PARSE_MODES,
|
|
4187
4450
|
RecordHandler,
|
|
4188
4451
|
Recorder,
|
|
4189
4452
|
RedactHandler,
|
|
4453
|
+
RedactionAudit,
|
|
4190
4454
|
Replay,
|
|
4191
4455
|
TOOL_KINDS,
|
|
4192
4456
|
applyLimit,
|