geovouch 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -97
- package/dist/index.d.ts +135 -0
- package/dist/index.js +94 -0
- package/package.json +22 -31
- package/dist/gemini.d.ts +0 -28
- package/dist/gemini.js +0 -62
- package/dist/matchers.d.ts +0 -32
- package/dist/matchers.js +0 -46
- package/dist/metrics.d.ts +0 -139
- package/dist/metrics.js +0 -273
- package/dist/runner.d.ts +0 -58
- package/dist/runner.js +0 -80
package/README.md
CHANGED
|
@@ -1,121 +1,54 @@
|
|
|
1
1
|
# geovouch
|
|
2
2
|
|
|
3
|
-
Measure whether AI assistants name your brand when buyers ask them questions —
|
|
4
|
-
measurement
|
|
5
|
-
|
|
6
|
-
Pure TypeScript. **Zero dependencies, zero IO.** Runs in a Cloudflare Worker, in Node, in a browser,
|
|
7
|
-
in a test with no database.
|
|
3
|
+
Measure whether AI assistants name your brand when buyers ask them questions — know when that
|
|
4
|
+
measurement can carry a decision, and prove whether what you shipped actually worked.
|
|
8
5
|
|
|
9
6
|
```bash
|
|
10
7
|
npm i geovouch
|
|
11
8
|
```
|
|
12
9
|
|
|
13
|
-
---
|
|
14
|
-
|
|
15
|
-
## The one idea
|
|
16
|
-
|
|
17
|
-
**One row per (prompt × engine × repetition). Never a metric.**
|
|
18
|
-
|
|
19
|
-
A grounded AI answer is non-deterministic. Two runs of the *same* prompt on the *same* engine within
|
|
20
|
-
24 hours share only 0.32–0.43 of their source set (St. Gallen, arXiv:2604.07585). So a single reading
|
|
21
|
-
per prompt is a coin flip, and a weekly dashboard number computed from one draw is noise with a
|
|
22
|
-
decimal point.
|
|
23
|
-
|
|
24
|
-
Everything here computes over a pooled window of atomic observations, and **never averages across
|
|
25
|
-
engines** — they diverge by ~46×, so one blended "AI visibility score" is an average of four different
|
|
26
|
-
instruments.
|
|
27
|
-
|
|
28
10
|
```ts
|
|
29
|
-
import {
|
|
30
|
-
|
|
31
|
-
const observations: GeoObservation[] = [
|
|
32
|
-
{ promptText: "best AI notetaker?", engine: "gemini", grounded: true,
|
|
33
|
-
brandMentioned: true, sourced: false, competitors: ["Otter"] },
|
|
34
|
-
// …one row per answer, pooled over your window
|
|
35
|
-
];
|
|
36
|
-
|
|
37
|
-
const w = computeGeoWindow(observations);
|
|
38
|
-
w.sovPct; // share of voice %, or null when nothing was named at all
|
|
39
|
-
w.sovCi; // { lo, hi } in pp — an observation-CLUSTER bootstrap, because one answer
|
|
40
|
-
// contributes several brand presences and they are not independent draws
|
|
41
|
-
w.presenceRateCi; // Wilson score interval — this one IS a Bernoulli rate
|
|
42
|
-
w.sourcedRateCi; // Wilson, over the GROUNDED observations only
|
|
43
|
-
w.lowConfidence; // true while any measured metric is wider than ±7pp
|
|
44
|
-
w.denominatorWidth; // distinct brands in the denominator — losing to 2 rivals ≠ losing to 20
|
|
45
|
-
w.observations; // the sample size. This IS the number that makes the rest readable.
|
|
46
|
-
```
|
|
47
|
-
|
|
48
|
-
`sovPct === null` means **nothing was named**, which is not a zero share. That distinction is load
|
|
49
|
-
bearing throughout: *unmeasured is never a zero.*
|
|
11
|
+
import { GeoVouch } from "geovouch";
|
|
12
|
+
const gv = new GeoVouch({ apiKey: process.env.GEOVOUCH_API_KEY! });
|
|
50
13
|
|
|
51
|
-
|
|
14
|
+
const { engines } = await gv.scoreboard(); // per engine, never averaged
|
|
15
|
+
const { actionable } = await gv.gaps(); // only what the evidence can carry
|
|
16
|
+
const brief = await gv.brief(actionable[0].question);
|
|
52
17
|
|
|
53
|
-
|
|
18
|
+
// …your agent writes and ships the page, in your stack …
|
|
54
19
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
```ts
|
|
58
|
-
import { evaluatePromptGap, PROMPT_ACTION_FWER } from "geovouch/metrics";
|
|
20
|
+
await gv.recordAction({ question: actionable[0].question, kind: "answer_page",
|
|
21
|
+
url: "https://you.com/blog/x", publishedAt: new Date() });
|
|
59
22
|
|
|
60
|
-
|
|
61
|
-
const decision = evaluatePromptGap(brandMentions, observations, 40);
|
|
62
|
-
|
|
63
|
-
decision.status; // "unmeasured" | "not_gap" | "research_only" | "proven_gap"
|
|
64
|
-
decision.reasons; // why it is not proven yet, in sentences
|
|
23
|
+
const { assets } = await gv.lift(); // did it work?
|
|
65
24
|
```
|
|
66
25
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
| condition | what it rules out |
|
|
70
|
-
|---|---|
|
|
71
|
-
| Wilson 95% interval, half-width ≤ 7pp | a point estimate that a few more draws would move |
|
|
72
|
-
| interval entirely below 50% | "we might be losing this question" |
|
|
73
|
-
| anytime-valid e-value ≥ family size ÷ α | the 40-hypotheses-one-looks-significant problem |
|
|
74
|
-
|
|
75
|
-
The third is the one most tools skip. It is a **one-sided finite-mixture Bernoulli e-process** against
|
|
76
|
-
the composite null *p ≥ 0.5*, so **repeated daily looks cost no alpha** (Ville's inequality) — you can
|
|
77
|
-
check every morning without inflating your error rate, which is exactly what a dashboard invites you
|
|
78
|
-
to do. The threshold is e-Bonferroni: `familySize / α`, so 40 hypotheses at α=0.05 needs an e-value of
|
|
79
|
-
**800**.
|
|
80
|
-
|
|
81
|
-
At n≈216 per cell, roughly a third of genuinely-low prompts clear it. That is the honest yield, and a
|
|
82
|
-
tool that reports more than that is reporting noise.
|
|
83
|
-
|
|
84
|
-
> A bank change is a **ruler** change. Bump your prompt-bank version when the bank changes, or the
|
|
85
|
-
> frozen hypothesis family silently grows past the denominator it is priced with.
|
|
26
|
+
## What this package deliberately does not expose
|
|
86
27
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
```ts
|
|
92
|
-
import { buildBrandMatcher, buildCompetitorMatchers, namesIn } from "geovouch/matchers";
|
|
93
|
-
import { groundedHosts, groundedQueryCount } from "geovouch/gemini";
|
|
94
|
-
import { runGeoScan } from "geovouch/runner";
|
|
95
|
-
```
|
|
28
|
+
No database, no migrations, no vendor token, no engine list to configure, no statistical parameter to
|
|
29
|
+
tune. Those are ours — and keeping them ours is what lets the numbers get better without you
|
|
30
|
+
redeploying.
|
|
96
31
|
|
|
97
|
-
|
|
98
|
-
of a CamelCase brand, which is how 93% of them actually appear.
|
|
99
|
-
- **`geovouch/gemini`** — parse Gemini grounding metadata into retrieved hosts and billable query
|
|
100
|
-
counts. A host must be domain-shaped; a redirector is not a publisher.
|
|
101
|
-
- **`geovouch/runner`** — the scan loop, with every engine and store injected as a **port**. No IO of
|
|
102
|
-
its own, so it is testable without a network.
|
|
32
|
+
## Three things the API refuses to blur
|
|
103
33
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
34
|
+
**`unmeasured` is never `0`.** An engine that answered without retrieving has told you nothing about
|
|
35
|
+
whether your pages were a source, so `sourcedRatePct` is `null` rather than zero. A lift verdict with
|
|
36
|
+
no control is `unmeasured`, not "no effect".
|
|
107
37
|
|
|
108
|
-
|
|
38
|
+
**Engines are never averaged.** They diverge by roughly 46×, so a single blended "AI visibility score"
|
|
39
|
+
is the mean of four different instruments.
|
|
109
40
|
|
|
110
|
-
|
|
41
|
+
**A gap you may act on is not a gap you can see.** `gaps()` returns two lists. `actionable` cleared a
|
|
42
|
+
precision floor, sat fully below the 50% win/loss line, and passed an anytime-valid familywise
|
|
43
|
+
threshold — so you can check every morning without inflating your error rate. `researchOnly` did not,
|
|
44
|
+
and spending on it is spending on a draw.
|
|
111
45
|
|
|
112
|
-
|
|
113
|
-
`sourced` asks whether one of *your* domains was among the URLs the engine actually **retrieved**, and
|
|
114
|
-
it is the only signal a model cannot produce from memory. It is `null`, never `false`, when an answer
|
|
115
|
-
did not ground: an engine that answered without searching has told you nothing about your retrieval.
|
|
46
|
+
## The loop
|
|
116
47
|
|
|
117
|
-
|
|
48
|
+
`scoreboard` → `gaps` → `brief` → **you act, in your own stack** → `recordAction` → `lift`
|
|
118
49
|
|
|
119
|
-
|
|
50
|
+
We never publish on your behalf: your agent already has the access, the conventions and the review
|
|
51
|
+
rules, and any connector we built would be a worse version of what you already own. The receipt is the
|
|
52
|
+
seam — it is what makes the last step possible for work we did not do.
|
|
120
53
|
|
|
121
54
|
MIT © The Influence Company
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
export type GeoVouchOptions = {
|
|
2
|
+
apiKey: string;
|
|
3
|
+
/** Override for self-hosted or preview deployments. */
|
|
4
|
+
baseUrl?: string;
|
|
5
|
+
/** Required only when your key covers several projects. A project-scoped key ignores it. */
|
|
6
|
+
project?: string;
|
|
7
|
+
fetchImpl?: typeof fetch;
|
|
8
|
+
};
|
|
9
|
+
/** Every failure arrives as this, with the server's own sentence. A thrown string would make callers
|
|
10
|
+
* parse prose to find out whether to retry. */
|
|
11
|
+
export declare class GeoVouchError extends Error {
|
|
12
|
+
readonly status: number;
|
|
13
|
+
readonly body: unknown;
|
|
14
|
+
constructor(status: number, message: string, body: unknown);
|
|
15
|
+
}
|
|
16
|
+
export declare class GeoVouch {
|
|
17
|
+
#private;
|
|
18
|
+
constructor(options: GeoVouchOptions);
|
|
19
|
+
/** Which projects this key can read. */
|
|
20
|
+
projects(): Promise<{
|
|
21
|
+
projects: {
|
|
22
|
+
id: string;
|
|
23
|
+
slug: string;
|
|
24
|
+
name: string;
|
|
25
|
+
scanEnabled: boolean;
|
|
26
|
+
}[];
|
|
27
|
+
keyScope: string;
|
|
28
|
+
}>;
|
|
29
|
+
/** Where you stand, per engine. Never averaged across engines — they diverge by ~46x, so one
|
|
30
|
+
* blended score is the mean of four different instruments. */
|
|
31
|
+
scoreboard(): Promise<ScoreboardResponse>;
|
|
32
|
+
/** The buyer questions you are losing, split by whether the evidence can carry a decision.
|
|
33
|
+
* `actionable` cleared precision, the 50% boundary and an anytime-valid familywise threshold;
|
|
34
|
+
* `researchOnly` did not, and acting on it is acting on a draw. */
|
|
35
|
+
gaps(): Promise<GapsResponse>;
|
|
36
|
+
/** What to write for one actionable question. */
|
|
37
|
+
brief(question: string): Promise<Record<string, unknown>>;
|
|
38
|
+
/** Ranked next-best-actions. An EMPTY list means measured-and-nothing-to-do; it never means the
|
|
39
|
+
* gate was shut — that arrives as a `collect_more_data` action naming the blocker. */
|
|
40
|
+
recommend(): Promise<{
|
|
41
|
+
actions: unknown[];
|
|
42
|
+
}>;
|
|
43
|
+
/** TELL US WHAT YOU DID. This is the verb that closes the loop: we never publish on your behalf, so
|
|
44
|
+
* without a receipt your work is invisible to the step that measures whether it landed.
|
|
45
|
+
*
|
|
46
|
+
* `publishedAt` is when the work went LIVE, not when you are reporting it — it is the before/after
|
|
47
|
+
* pivot, and reporting Friday about a Tuesday page would count three days of "after" as before. */
|
|
48
|
+
recordAction(input: {
|
|
49
|
+
question: string;
|
|
50
|
+
kind: ActionKind;
|
|
51
|
+
url?: string;
|
|
52
|
+
publishedAt: Date | string;
|
|
53
|
+
note?: string;
|
|
54
|
+
}): Promise<{
|
|
55
|
+
action: unknown;
|
|
56
|
+
next: string;
|
|
57
|
+
}>;
|
|
58
|
+
/** Did it work? Control-adjusted, per engine, against a holdout arm nothing ever targets.
|
|
59
|
+
* `unmeasured` means a window is missing or there is no control — it is not a zero. */
|
|
60
|
+
lift(): Promise<LiftResponse>;
|
|
61
|
+
}
|
|
62
|
+
export type ActionKind = "answer_page" | "page_strengthened" | "structured_data" | "docs_page" | "community_answer" | "third_party_listicle" | "backlink";
|
|
63
|
+
export type EngineScore = {
|
|
64
|
+
engine: string;
|
|
65
|
+
status: "measured" | "unmeasured";
|
|
66
|
+
observations: number;
|
|
67
|
+
shareOfVoicePct?: number | null;
|
|
68
|
+
presenceRatePct?: number | null;
|
|
69
|
+
/** NULL, never 0, when nothing grounded: an engine that answered without retrieving has told you
|
|
70
|
+
* nothing about whether your pages were a source. */
|
|
71
|
+
sourcedRatePct?: number | null;
|
|
72
|
+
confidence95?: {
|
|
73
|
+
shareOfVoicePct: {
|
|
74
|
+
low: number;
|
|
75
|
+
high: number;
|
|
76
|
+
} | null;
|
|
77
|
+
presenceRatePct: {
|
|
78
|
+
low: number;
|
|
79
|
+
high: number;
|
|
80
|
+
} | null;
|
|
81
|
+
};
|
|
82
|
+
lowConfidence?: boolean;
|
|
83
|
+
rivalsNamed?: number;
|
|
84
|
+
topRivals?: {
|
|
85
|
+
label: string;
|
|
86
|
+
count: number;
|
|
87
|
+
}[];
|
|
88
|
+
};
|
|
89
|
+
export type Readiness = {
|
|
90
|
+
actionable: boolean;
|
|
91
|
+
reasons: string[];
|
|
92
|
+
};
|
|
93
|
+
export type ProjectRef = {
|
|
94
|
+
id: string;
|
|
95
|
+
slug: string;
|
|
96
|
+
name: string;
|
|
97
|
+
};
|
|
98
|
+
export type ScoreboardResponse = {
|
|
99
|
+
project: ProjectRef;
|
|
100
|
+
readiness: Readiness;
|
|
101
|
+
engines: EngineScore[];
|
|
102
|
+
};
|
|
103
|
+
export type GapsResponse = {
|
|
104
|
+
project: ProjectRef;
|
|
105
|
+
readiness: Readiness;
|
|
106
|
+
actionable: Gap[];
|
|
107
|
+
researchOnly: Gap[];
|
|
108
|
+
};
|
|
109
|
+
export type Gap = {
|
|
110
|
+
question: string;
|
|
111
|
+
actionable: boolean;
|
|
112
|
+
citedRate?: number | null;
|
|
113
|
+
researchReasons?: string[];
|
|
114
|
+
competitors?: string[];
|
|
115
|
+
};
|
|
116
|
+
export type LiftResponse = {
|
|
117
|
+
project: ProjectRef;
|
|
118
|
+
holdoutArm: string[];
|
|
119
|
+
assets: {
|
|
120
|
+
slug: string;
|
|
121
|
+
question: string;
|
|
122
|
+
publishedAt: string;
|
|
123
|
+
/** Non-null means the after-window is not full yet. That resolves with TIME, not more content —
|
|
124
|
+
* the distinction between "no effect" and "too early to say". */
|
|
125
|
+
daysUntilComparable: number | null;
|
|
126
|
+
engines: {
|
|
127
|
+
engine: string;
|
|
128
|
+
status: string;
|
|
129
|
+
proven: boolean;
|
|
130
|
+
netCiteLift: number | null;
|
|
131
|
+
verdict: string;
|
|
132
|
+
}[];
|
|
133
|
+
}[];
|
|
134
|
+
guidance: string;
|
|
135
|
+
};
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/** Every failure arrives as this, with the server's own sentence. A thrown string would make callers
|
|
2
|
+
* parse prose to find out whether to retry. */
|
|
3
|
+
export class GeoVouchError extends Error {
|
|
4
|
+
status;
|
|
5
|
+
body;
|
|
6
|
+
constructor(status, message, body) {
|
|
7
|
+
super(message);
|
|
8
|
+
this.status = status;
|
|
9
|
+
this.body = body;
|
|
10
|
+
this.name = "GeoVouchError";
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
const DEFAULT_BASE = "https://geovouch.zjudn2013.workers.dev";
|
|
14
|
+
export class GeoVouch {
|
|
15
|
+
#key;
|
|
16
|
+
#base;
|
|
17
|
+
#project;
|
|
18
|
+
#fetch;
|
|
19
|
+
constructor(options) {
|
|
20
|
+
if (!options.apiKey)
|
|
21
|
+
throw new Error("geovouch: apiKey is required");
|
|
22
|
+
this.#key = options.apiKey;
|
|
23
|
+
this.#base = (options.baseUrl ?? DEFAULT_BASE).replace(/\/+$/, "");
|
|
24
|
+
this.#project = options.project;
|
|
25
|
+
this.#fetch = options.fetchImpl ?? fetch;
|
|
26
|
+
}
|
|
27
|
+
async #call(path, init) {
|
|
28
|
+
const url = new URL(`${this.#base}/api/v1/${path}`);
|
|
29
|
+
if (this.#project)
|
|
30
|
+
url.searchParams.set("project", this.#project);
|
|
31
|
+
for (const [k, v] of Object.entries(init?.query ?? {}))
|
|
32
|
+
url.searchParams.set(k, v);
|
|
33
|
+
const res = await this.#fetch(url.toString(), {
|
|
34
|
+
method: init?.method ?? "GET",
|
|
35
|
+
// Bearer only. A key in a query string lands in access logs, proxy logs, browser history and
|
|
36
|
+
// Referer headers — none of which anyone thinks to rotate.
|
|
37
|
+
headers: { authorization: `Bearer ${this.#key}`, ...(init?.body ? { "content-type": "application/json" } : {}) },
|
|
38
|
+
body: init?.body ? JSON.stringify(init.body) : undefined,
|
|
39
|
+
});
|
|
40
|
+
const text = await res.text();
|
|
41
|
+
let parsed;
|
|
42
|
+
try {
|
|
43
|
+
parsed = text ? JSON.parse(text) : null;
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
throw new GeoVouchError(res.status, `geovouch: ${res.status} returned a non-JSON body`, text);
|
|
47
|
+
}
|
|
48
|
+
if (!res.ok) {
|
|
49
|
+
const message = parsed?.error;
|
|
50
|
+
throw new GeoVouchError(res.status, message || `geovouch: request failed (${res.status})`, parsed);
|
|
51
|
+
}
|
|
52
|
+
return parsed;
|
|
53
|
+
}
|
|
54
|
+
/** Which projects this key can read. */
|
|
55
|
+
projects() {
|
|
56
|
+
return this.#call("projects");
|
|
57
|
+
}
|
|
58
|
+
/** Where you stand, per engine. Never averaged across engines — they diverge by ~46x, so one
|
|
59
|
+
* blended score is the mean of four different instruments. */
|
|
60
|
+
scoreboard() {
|
|
61
|
+
return this.#call("scoreboard");
|
|
62
|
+
}
|
|
63
|
+
/** The buyer questions you are losing, split by whether the evidence can carry a decision.
|
|
64
|
+
* `actionable` cleared precision, the 50% boundary and an anytime-valid familywise threshold;
|
|
65
|
+
* `researchOnly` did not, and acting on it is acting on a draw. */
|
|
66
|
+
gaps() {
|
|
67
|
+
return this.#call("gaps");
|
|
68
|
+
}
|
|
69
|
+
/** What to write for one actionable question. */
|
|
70
|
+
brief(question) {
|
|
71
|
+
return this.#call("brief", { query: { question } });
|
|
72
|
+
}
|
|
73
|
+
/** Ranked next-best-actions. An EMPTY list means measured-and-nothing-to-do; it never means the
|
|
74
|
+
* gate was shut — that arrives as a `collect_more_data` action naming the blocker. */
|
|
75
|
+
recommend() {
|
|
76
|
+
return this.#call("recommend");
|
|
77
|
+
}
|
|
78
|
+
/** TELL US WHAT YOU DID. This is the verb that closes the loop: we never publish on your behalf, so
|
|
79
|
+
* without a receipt your work is invisible to the step that measures whether it landed.
|
|
80
|
+
*
|
|
81
|
+
* `publishedAt` is when the work went LIVE, not when you are reporting it — it is the before/after
|
|
82
|
+
* pivot, and reporting Friday about a Tuesday page would count three days of "after" as before. */
|
|
83
|
+
recordAction(input) {
|
|
84
|
+
return this.#call("actions", {
|
|
85
|
+
method: "POST",
|
|
86
|
+
body: { ...input, publishedAt: input.publishedAt instanceof Date ? input.publishedAt.toISOString() : input.publishedAt },
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
/** Did it work? Control-adjusted, per engine, against a holdout arm nothing ever targets.
|
|
90
|
+
* `unmeasured` means a window is missing or there is no control — it is not a zero. */
|
|
91
|
+
lift() {
|
|
92
|
+
return this.#call("lift");
|
|
93
|
+
}
|
|
94
|
+
}
|
package/package.json
CHANGED
|
@@ -1,37 +1,28 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "geovouch",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "GeoVouch
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"description": "GeoVouch client — measure whether AI assistants name your brand, know when the evidence can carry a decision, and prove whether what you shipped worked. Bring an API key; the storage, the engines and the statistics are ours.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"engines": {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
"default": "./dist/runner.js"
|
|
29
|
-
}
|
|
30
|
-
},
|
|
31
|
-
"scripts": {
|
|
32
|
-
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
33
|
-
"build": "tsc -p tsconfig.build.json",
|
|
34
|
-
"test": "node --test --experimental-transform-types test/*.test.ts",
|
|
35
|
-
"boundary": "node ../../scripts/check-boundary.mjs"
|
|
36
|
-
}
|
|
8
|
+
"node": ">=20"
|
|
9
|
+
},
|
|
10
|
+
"files": ["dist", "README.md", "LICENSE"],
|
|
11
|
+
"repository": {
|
|
12
|
+
"type": "git",
|
|
13
|
+
"url": "git+https://github.com/theinfluencecompany/geovouch.git",
|
|
14
|
+
"directory": "libs/client"
|
|
15
|
+
},
|
|
16
|
+
"keywords": ["geo", "aeo", "generative-engine-optimization", "ai-search", "share-of-voice", "llm", "brand-visibility"],
|
|
17
|
+
"exports": {
|
|
18
|
+
".": {
|
|
19
|
+
"types": "./dist/index.d.ts",
|
|
20
|
+
"default": "./dist/index.js"
|
|
21
|
+
}
|
|
22
|
+
},
|
|
23
|
+
"scripts": {
|
|
24
|
+
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
25
|
+
"build": "tsc -p tsconfig.build.json",
|
|
26
|
+
"test": "node --test --experimental-transform-types test/*.test.ts"
|
|
27
|
+
}
|
|
37
28
|
}
|
package/dist/gemini.d.ts
DELETED
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
/** Google's opaque grounding redirect. EVERY `web.uri` is one of these, so it is never a source — a
|
|
2
|
-
* set containing only this host means "we read the wrong field", not "Google cited itself". */
|
|
3
|
-
export declare const GROUNDING_REDIRECT_HOST = "vertexaisearch.cloud.google.com";
|
|
4
|
-
/** Hostnames the engine retrieved, off Gemini's grounding metadata. Empty when the answer did not
|
|
5
|
-
* ground.
|
|
6
|
-
*
|
|
7
|
-
* READ `web.title`, NOT `web.uri`, AND NEVER `web.domain`. Measured: every chunk is
|
|
8
|
-
* `{web:{uri,title}}`, `web.domain` is sent ZERO times — and @ai-sdk/google declares
|
|
9
|
-
* `web: z.object({uri, title})`, so zod would strip a future `domain` before it reached us; a
|
|
10
|
-
* fallback to it would read as a safety net while being unreachable code. Every `uri` is the opaque
|
|
11
|
-
* redirect above. The publisher is in `title`, which on the Developer API is a bare domain — but not
|
|
12
|
-
* contractually (Vertex puts page titles there), so a title is accepted only when it LOOKS like a
|
|
13
|
-
* hostname; "Best AI apps | Some Site" is dropped rather than URL-parsed into garbage. */
|
|
14
|
-
export declare function groundedHosts(providerMetadata: unknown): string[];
|
|
15
|
-
/** Search queries the grounded answer actually executed — the unit Gemini 3.x BILLS on. The model
|
|
16
|
-
* decides the fan-out per prompt (may be zero: a memory answer inside a grounded request), so a
|
|
17
|
-
* prompt bank of a known size is NOT a bill of a known size; counting the answer is the only bound. */
|
|
18
|
-
export declare function groundedQueryCount(providerMetadata: unknown): number;
|
|
19
|
-
/** $14 per 1,000 search queries — ai.google.dev/gemini-api/docs/pricing, read 2026-09-01. The 2.5
|
|
20
|
-
* line billed per grounded PROMPT; the 3.x line bills per SEARCH QUERY. The 5,000-free-per-month
|
|
21
|
-
* allowance is deliberately NOT modelled: it is shared across every Gemini caller on an account, so
|
|
22
|
-
* a second grounded call site would silently make the number wrong. Charging from the first query is
|
|
23
|
-
* conservative and cannot drift. */
|
|
24
|
-
export declare const GROUNDED_QUERY_CENTS = 1.4;
|
|
25
|
-
/** What a run's grounded searches cost, in USD. The rate lives here and nowhere else — a console that
|
|
26
|
-
* multiplied the query count by its own copy of the price would go quietly wrong the day Google
|
|
27
|
-
* moves it. */
|
|
28
|
-
export declare const groundedCostUsd: (searchQueries: number) => number;
|
package/dist/gemini.js
DELETED
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
// geovouch/gemini — grounding-metadata parsing for the Gemini Developer API. Product-agnostic: see
|
|
2
|
-
// BOUNDARY.md. Every claim here was measured against the live API on 2026-09-04 (49 grounding chunks
|
|
3
|
-
// across three calls through `ai` + `@ai-sdk/google`), because the first consumer shipped two months
|
|
4
|
-
// of dead data by trusting a field the API never sends.
|
|
5
|
-
/** Google's opaque grounding redirect. EVERY `web.uri` is one of these, so it is never a source — a
|
|
6
|
-
* set containing only this host means "we read the wrong field", not "Google cited itself". */
|
|
7
|
-
export const GROUNDING_REDIRECT_HOST = "vertexaisearch.cloud.google.com";
|
|
8
|
-
/** Hostnames the engine retrieved, off Gemini's grounding metadata. Empty when the answer did not
|
|
9
|
-
* ground.
|
|
10
|
-
*
|
|
11
|
-
* READ `web.title`, NOT `web.uri`, AND NEVER `web.domain`. Measured: every chunk is
|
|
12
|
-
* `{web:{uri,title}}`, `web.domain` is sent ZERO times — and @ai-sdk/google declares
|
|
13
|
-
* `web: z.object({uri, title})`, so zod would strip a future `domain` before it reached us; a
|
|
14
|
-
* fallback to it would read as a safety net while being unreachable code. Every `uri` is the opaque
|
|
15
|
-
* redirect above. The publisher is in `title`, which on the Developer API is a bare domain — but not
|
|
16
|
-
* contractually (Vertex puts page titles there), so a title is accepted only when it LOOKS like a
|
|
17
|
-
* hostname; "Best AI apps | Some Site" is dropped rather than URL-parsed into garbage. */
|
|
18
|
-
export function groundedHosts(providerMetadata) {
|
|
19
|
-
const meta = providerMetadata;
|
|
20
|
-
const chunks = meta?.google?.groundingMetadata?.groundingChunks ?? [];
|
|
21
|
-
const hosts = new Set();
|
|
22
|
-
for (const c of chunks) {
|
|
23
|
-
const host = hostFromTitle(c?.web?.title) ?? hostFromUri(c?.web?.uri);
|
|
24
|
-
if (host && host !== GROUNDING_REDIRECT_HOST)
|
|
25
|
-
hosts.add(host);
|
|
26
|
-
}
|
|
27
|
-
return [...hosts];
|
|
28
|
-
}
|
|
29
|
-
/** A bare hostname (`example.ai`, `www.reddit.com`) → normalized; anything else → null. */
|
|
30
|
-
function hostFromTitle(title) {
|
|
31
|
-
const t = title?.trim().toLowerCase();
|
|
32
|
-
if (!t || !/^[a-z0-9-]+(\.[a-z0-9-]+)+$/.test(t))
|
|
33
|
-
return null;
|
|
34
|
-
return t.replace(/^www\./, "");
|
|
35
|
-
}
|
|
36
|
-
function hostFromUri(uri) {
|
|
37
|
-
if (!uri)
|
|
38
|
-
return null;
|
|
39
|
-
try {
|
|
40
|
-
return new URL(uri.startsWith("http") ? uri : `https://${uri}`).hostname.replace(/^www\./, "");
|
|
41
|
-
}
|
|
42
|
-
catch {
|
|
43
|
-
return null;
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
/** Search queries the grounded answer actually executed — the unit Gemini 3.x BILLS on. The model
|
|
47
|
-
* decides the fan-out per prompt (may be zero: a memory answer inside a grounded request), so a
|
|
48
|
-
* prompt bank of a known size is NOT a bill of a known size; counting the answer is the only bound. */
|
|
49
|
-
export function groundedQueryCount(providerMetadata) {
|
|
50
|
-
const meta = providerMetadata;
|
|
51
|
-
return meta?.google?.groundingMetadata?.webSearchQueries?.length ?? 0;
|
|
52
|
-
}
|
|
53
|
-
/** $14 per 1,000 search queries — ai.google.dev/gemini-api/docs/pricing, read 2026-09-01. The 2.5
|
|
54
|
-
* line billed per grounded PROMPT; the 3.x line bills per SEARCH QUERY. The 5,000-free-per-month
|
|
55
|
-
* allowance is deliberately NOT modelled: it is shared across every Gemini caller on an account, so
|
|
56
|
-
* a second grounded call site would silently make the number wrong. Charging from the first query is
|
|
57
|
-
* conservative and cannot drift. */
|
|
58
|
-
export const GROUNDED_QUERY_CENTS = 1.4;
|
|
59
|
-
/** What a run's grounded searches cost, in USD. The rate lives here and nowhere else — a console that
|
|
60
|
-
* multiplied the query count by its own copy of the price would go quietly wrong the day Google
|
|
61
|
-
* moves it. */
|
|
62
|
-
export const groundedCostUsd = (searchQueries) => Math.round(searchQueries * GROUNDED_QUERY_CENTS) / 100;
|
package/dist/matchers.d.ts
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
export type BrandConfig = {
|
|
2
|
-
/** The brand word as engines write it, e.g. "Acme". Matched word-bounded, case-insensitive. */
|
|
3
|
-
name: string;
|
|
4
|
-
/** Registrable domains that count as the brand's own pages among retrieved sources ("acme.ai"
|
|
5
|
-
* matches itself and any subdomain). */
|
|
6
|
-
domains: readonly string[];
|
|
7
|
-
/** Extra spellings that also count as a mention (a common misspelling, a former name). */
|
|
8
|
-
aliases?: readonly string[];
|
|
9
|
-
};
|
|
10
|
-
export type BrandMatcher = {
|
|
11
|
-
/** Was the brand NAMED in the prose? A model can satisfy this from memory or confabulation — pair
|
|
12
|
-
* it with `sourcedHost` on retrieved sources, which it cannot fake. */
|
|
13
|
-
mentioned(answer: string): boolean;
|
|
14
|
-
/** Is this retrieved host one of ours? */
|
|
15
|
-
sourcedHost(host: string): boolean;
|
|
16
|
-
};
|
|
17
|
-
export declare function buildBrandMatcher(config: BrandConfig): BrandMatcher;
|
|
18
|
-
export type CompetitorMatcher = {
|
|
19
|
-
label: string;
|
|
20
|
-
re: RegExp;
|
|
21
|
-
};
|
|
22
|
-
/** Build presence matchers for a competitor set. Tokenizes labels at BOTH kinds of boundary — the
|
|
23
|
-
* separators an engine may drop, and the CamelCase seam an all-together label hides — then rejoins
|
|
24
|
-
* with an optional separator: "AcmeAI" → /\bAcme[.\s-]?AI\b/i, matching "Acme AI", "AcmeAI" and
|
|
25
|
-
* "Acme.AI" alike. The FULL domain is the second alternative, never its root. */
|
|
26
|
-
export declare function buildCompetitorMatchers(competitors: readonly {
|
|
27
|
-
domain: string;
|
|
28
|
-
label: string;
|
|
29
|
-
}[]): CompetitorMatcher[];
|
|
30
|
-
/** Competitor labels named in one answer — the atoms of share-of-voice's denominator. A false
|
|
31
|
-
* positive here is indistinguishable from the measured brand losing ground. */
|
|
32
|
-
export declare function namesIn(answer: string, matchers: readonly CompetitorMatcher[]): string[];
|
package/dist/matchers.js
DELETED
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
// geovouch/matchers — brand + competitor presence detection over engine answer text. Product-agnostic:
|
|
2
|
-
// see BOUNDARY.md. Every rule here was paid for by a measured incident on the first consumer, so the
|
|
3
|
-
// rules travel with the code:
|
|
4
|
-
// * A competitor is matched by LABEL (separator- AND CamelCase-flexible) or FULL DOMAIN — never a
|
|
5
|
-
// bare domain root. A rival on a `<noun>.ai` domain whose root is an everyday word the measured
|
|
6
|
-
// product's own copy uses constantly once matched as `\b<noun>\b` and scored a presence on 8/10
|
|
7
|
-
// answers against a true 5/10 — and every false positive is a point in share-of-voice's
|
|
8
|
-
// DENOMINATOR, i.e. it pushes the measured brand DOWN.
|
|
9
|
-
// * CamelCase must tokenize, because engines write the spaced form: a stored one-token label
|
|
10
|
-
// ("AcmeAI") against the written spaced form ("Acme AI") missed ~93% of real namings
|
|
11
|
-
// (1 of 15 measured).
|
|
12
|
-
// * "AI" never splits internally — the seam is lower/digit → UPPER, so consecutive capitals stay
|
|
13
|
-
// whole and a label like "PopUp" does not become a match for the ordinary phrase "pop up".
|
|
14
|
-
const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
15
|
-
export function buildBrandMatcher(config) {
|
|
16
|
-
const spellings = [config.name, ...(config.aliases ?? [])].filter(Boolean).map(escapeRe);
|
|
17
|
-
const re = new RegExp(`\\b(?:${spellings.join("|")})\\b`, "i");
|
|
18
|
-
const domains = config.domains.map((d) => d.toLowerCase());
|
|
19
|
-
return {
|
|
20
|
-
mentioned: (answer) => re.test(answer),
|
|
21
|
-
sourcedHost: (host) => {
|
|
22
|
-
const h = host.toLowerCase();
|
|
23
|
-
return domains.some((d) => h === d || h.endsWith(`.${d}`));
|
|
24
|
-
},
|
|
25
|
-
};
|
|
26
|
-
}
|
|
27
|
-
/** Build presence matchers for a competitor set. Tokenizes labels at BOTH kinds of boundary — the
|
|
28
|
-
* separators an engine may drop, and the CamelCase seam an all-together label hides — then rejoins
|
|
29
|
-
* with an optional separator: "AcmeAI" → /\bAcme[.\s-]?AI\b/i, matching "Acme AI", "AcmeAI" and
|
|
30
|
-
* "Acme.AI" alike. The FULL domain is the second alternative, never its root. */
|
|
31
|
-
export function buildCompetitorMatchers(competitors) {
|
|
32
|
-
return competitors.map((c) => {
|
|
33
|
-
const tokens = c.label
|
|
34
|
-
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
|
35
|
-
.split(/[.\s-]+/)
|
|
36
|
-
.filter(Boolean)
|
|
37
|
-
.map(escapeRe);
|
|
38
|
-
const label = tokens.join("[.\\s-]?");
|
|
39
|
-
return { label: c.label, re: new RegExp(`\\b${label}\\b|\\b${escapeRe(c.domain)}\\b`, "i") };
|
|
40
|
-
});
|
|
41
|
-
}
|
|
42
|
-
/** Competitor labels named in one answer — the atoms of share-of-voice's denominator. A false
|
|
43
|
-
* positive here is indistinguishable from the measured brand losing ground. */
|
|
44
|
-
export function namesIn(answer, matchers) {
|
|
45
|
-
return matchers.filter((c) => c.re.test(answer)).map((c) => c.label);
|
|
46
|
-
}
|
package/dist/metrics.d.ts
DELETED
|
@@ -1,139 +0,0 @@
|
|
|
1
|
-
/** One atomic observation: a single answer to one prompt on one engine. */
|
|
2
|
-
export type GeoObservation = {
|
|
3
|
-
promptText: string;
|
|
4
|
-
engine: string;
|
|
5
|
-
/** Did this answer actually retrieve? The denominator for `sourced`. */
|
|
6
|
-
grounded: boolean;
|
|
7
|
-
/** Was the measured brand NAMED in the prose? */
|
|
8
|
-
brandMentioned: boolean;
|
|
9
|
-
/** Was one of the brand's own domains among the retrieved sources? null when the answer did not
|
|
10
|
-
* ground. */
|
|
11
|
-
sourced: boolean | null;
|
|
12
|
-
/** Competitor labels named in this one answer. */
|
|
13
|
-
competitors: string[];
|
|
14
|
-
};
|
|
15
|
-
export type GeoWindowSummary = {
|
|
16
|
-
/** Share of voice %: brand presences / (brand + competitor presences), 0-100. NULL when the
|
|
17
|
-
* denominator is empty — nothing was named at all, which is not a zero share. Rounded to 0.1 for
|
|
18
|
-
* storage AFTER the internal share is computed at full precision (Elmo's rule — an intermediate
|
|
19
|
-
* integer percentage rounds small shares away). */
|
|
20
|
-
sovPct: number | null;
|
|
21
|
-
/** Presence rate %: observations naming the brand / total observations, 0-100. NULL when empty. */
|
|
22
|
-
presenceRatePct: number | null;
|
|
23
|
-
/** Sourced rate %: GROUNDED observations where a brand domain was a source / grounded observations.
|
|
24
|
-
* The metric a model cannot fake. NULL when nothing grounded — you cannot be a source of nothing. */
|
|
25
|
-
sourcedRatePct: number | null;
|
|
26
|
-
/** DISTINCT brands named at least once (the measured brand counts only if it appeared). The
|
|
27
|
-
* denominator's WIDTH — a low SoV against 2 rivals and against 20 are different facts. */
|
|
28
|
-
denominatorWidth: number;
|
|
29
|
-
/** Atomic observations pooled = sample size. 0 ⇒ measured nothing. */
|
|
30
|
-
observations: number;
|
|
31
|
-
/** Observations that actually grounded — the honest sourced denominator. */
|
|
32
|
-
groundedObservations: number;
|
|
33
|
-
/** Competitor labels by presence share, most-present first. Counts, not rates, so a reader can see
|
|
34
|
-
* the raw tally. */
|
|
35
|
-
competitors: {
|
|
36
|
-
label: string;
|
|
37
|
-
count: number;
|
|
38
|
-
}[];
|
|
39
|
-
/** 95% confidence intervals (in %) for the three metrics: observation-cluster bootstrap for SoV,
|
|
40
|
-
* Wilson score intervals for the two Bernoulli rates. A number without its interval is a claim
|
|
41
|
-
* without its uncertainty; a thin sample shows a WIDE interval, which is the honest signal. NULL
|
|
42
|
-
* when the metric's denominator is empty. */
|
|
43
|
-
sovCi: Interval | null;
|
|
44
|
-
presenceRateCi: Interval | null;
|
|
45
|
-
sourcedRateCi: Interval | null;
|
|
46
|
-
/** True when ANY measured headline metric is too imprecise: its farthest 95% bound lies more than
|
|
47
|
-
* `MAX_CI_HALF_WIDTH_PP` from the point estimate. A low-confidence number must be shown as a RANGE,
|
|
48
|
-
* never as a headline figure, and a lift claim inside it is unprovable. */
|
|
49
|
-
lowConfidence: boolean;
|
|
50
|
-
};
|
|
51
|
-
/** A confidence interval in percent (0-100), low/high bounds. */
|
|
52
|
-
export type Interval = {
|
|
53
|
-
lo: number;
|
|
54
|
-
hi: number;
|
|
55
|
-
};
|
|
56
|
-
/** The field's replicated noise floor is roughly ±5–7pp. A point estimate is too imprecise to
|
|
57
|
-
* headline when its 95% CI has a half-width above 7pp (14pp full width). */
|
|
58
|
-
export declare const MAX_CI_HALF_WIDTH_PP = 7;
|
|
59
|
-
export declare const MAX_CI_WIDTH_PP: number;
|
|
60
|
-
/** An intervention smaller than 7pp is not practically distinguishable from ordinary run-to-run
|
|
61
|
-
* movement even if a very large sample eventually makes it statistically significant. */
|
|
62
|
-
export declare const MIN_LIFT_PP = 7;
|
|
63
|
-
/** Familywise false-action budget for a frozen prompt-bank version. Repeated looks are handled by the
|
|
64
|
-
* e-process itself; this alpha is divided only across the fixed prompt × engine hypothesis family. */
|
|
65
|
-
export declare const PROMPT_ACTION_FWER = 0.05;
|
|
66
|
-
export type PromptGapEvidence = {
|
|
67
|
-
/** Current mixture e-value. It may fall after a previous crossing; acting on the first crossing is
|
|
68
|
-
* still protected by Ville's inequality. */
|
|
69
|
-
eValue: number;
|
|
70
|
-
/** e-Bonferroni threshold = fixed hypothesis-family size / familywise alpha. */
|
|
71
|
-
threshold: number;
|
|
72
|
-
hypothesisCount: number;
|
|
73
|
-
familywiseAlpha: number;
|
|
74
|
-
proven: boolean;
|
|
75
|
-
};
|
|
76
|
-
/** Anytime-valid evidence that one prompt-engine mention rate is below 50%. The observation stream must
|
|
77
|
-
* be scoped to ONE frozen prompt-bank version and must include only complete scans. Repeated dashboard,
|
|
78
|
-
* MCP and cron looks spend no extra alpha; e-Bonferroni controls the fixed prompt × engine family under
|
|
79
|
-
* arbitrary dependence. Wilson intervals remain a separate display/precision rule. */
|
|
80
|
-
export declare function promptGapEvidence(brandMentions: number, observations: number, hypothesisCount: number, familywiseAlpha?: number): PromptGapEvidence;
|
|
81
|
-
/** Unrounded Wilson bounds in percentage points. Contrast estimators consume these directly so
|
|
82
|
-
* intermediate rounding cannot move a final lift interval across zero. */
|
|
83
|
-
export declare function wilsonCiBounds(successes: number, total: number, z?: number): Interval | null;
|
|
84
|
-
/** 95% Wilson score interval for a binomial proportion `successes/total`, returned in PERCENT and
|
|
85
|
-
* rounded for display/storage. NULL when total=0. Wilson behaves correctly at small n and 0%/100%. */
|
|
86
|
-
export declare function wilsonCi(successes: number, total: number, z?: number): Interval | null;
|
|
87
|
-
/** Full width of an interval in pp; Infinity for a null (unmeasured) interval. */
|
|
88
|
-
export declare const ciWidth: (ci: Interval | null) => number;
|
|
89
|
-
/** The field threshold is about uncertainty around THE estimate, not half of an asymmetric interval's
|
|
90
|
-
* full width. Wilson intervals near 0%/100% are one-sided, so use the farther endpoint. */
|
|
91
|
-
export declare const ciMargin: (estimate: number | null, ci: Interval | null) => number;
|
|
92
|
-
export type PromptGapDecision = {
|
|
93
|
-
status: "unmeasured" | "not_gap" | "research_only" | "proven_gap";
|
|
94
|
-
citedRate: number | null;
|
|
95
|
-
citedRateCi: Interval | null;
|
|
96
|
-
confidenceReady: boolean;
|
|
97
|
-
evidence: PromptGapEvidence;
|
|
98
|
-
reasons: string[];
|
|
99
|
-
};
|
|
100
|
-
/** The one action gate for a prompt on one engine. The Wilson interval answers whether the estimate is
|
|
101
|
-
* precise enough to explain; the e-process answers whether repeated looks plus the fixed family still
|
|
102
|
-
* permit action. Callers must supply the FULL frozen prompt × engine family size, not the number of
|
|
103
|
-
* rows that happened to answer. */
|
|
104
|
-
export declare function evaluatePromptGap(brandMentions: number, observations: number, hypothesisCount: number, familywiseAlpha?: number): PromptGapDecision;
|
|
105
|
-
/** Deterministic cluster bootstrap for share of voice. One answer can name several rivals, so its SoV
|
|
106
|
-
* contributions are not Bernoulli trials and a Wilson interval over raw brand+rival mentions would
|
|
107
|
-
* be falsely narrow. Resampling whole observations preserves that within-answer clustering. A
|
|
108
|
-
* cluster-effective Wilson floor prevents the percentile bootstrap's known zero-width failure when
|
|
109
|
-
* a small sample lands entirely at 0% or 100%. */
|
|
110
|
-
export declare function bootstrapSovCi(observations: readonly GeoObservation[], samples?: number): Interval | null;
|
|
111
|
-
/** Aggregate a window of observations into one summary. Pure. Elmo's `computeReportMetrics` shape:
|
|
112
|
-
* share = brand / (brand + competitors), presence = brand / total, sourced = sourced / grounded, each
|
|
113
|
-
* with an explicit empty-denominator → null (never a divide, never a misleading 0). */
|
|
114
|
-
export declare function computeGeoWindow(observations: readonly GeoObservation[]): GeoWindowSummary;
|
|
115
|
-
/** Per-prompt aggregation for the ACTIVE LOOP: which buyer questions the brand is losing, and to whom,
|
|
116
|
-
* over the window. A prompt is a "gap" when the brand is named in a MINORITY of its observations —
|
|
117
|
-
* pooled, not a single draw. Ranked worst-first: most-uncited, then most-competitors-present. */
|
|
118
|
-
export type GeoPromptWindow = {
|
|
119
|
-
promptText: string;
|
|
120
|
-
observations: number;
|
|
121
|
-
brandMentions: number;
|
|
122
|
-
/** 0-1: brandMentions / observations. Below 0.5 ⇒ we lose this question more often than we win it. */
|
|
123
|
-
citedRate: number;
|
|
124
|
-
/** 95% Wilson interval in 0-1 units for agent-facing ranking and confidence gating. */
|
|
125
|
-
citedRateCi: {
|
|
126
|
-
lo: number;
|
|
127
|
-
hi: number;
|
|
128
|
-
};
|
|
129
|
-
/** True when the rate is wider than the headline precision floor OR its interval crosses the 50%
|
|
130
|
-
* win/loss boundary. A point estimate below 50% is not an actionable loss until its upper bound is
|
|
131
|
-
* also below 50%. */
|
|
132
|
-
lowConfidence: boolean;
|
|
133
|
-
/** Competitor labels named across this prompt's window, by frequency. */
|
|
134
|
-
competitors: {
|
|
135
|
-
label: string;
|
|
136
|
-
count: number;
|
|
137
|
-
}[];
|
|
138
|
-
};
|
|
139
|
-
export declare function computeGeoPromptWindows(observations: readonly GeoObservation[]): GeoPromptWindow[];
|
package/dist/metrics.js
DELETED
|
@@ -1,273 +0,0 @@
|
|
|
1
|
-
// geovouch/metrics — the pure math over atomic prompt-run observations. Product-agnostic: see BOUNDARY.md.
|
|
2
|
-
//
|
|
3
|
-
// PORTED FROM Elmo (elmohq/elmo, MIT — packages/lib/src/report-metrics.ts + apps/web/src/lib/
|
|
4
|
-
// visibility-stats.ts), whose design centre is exactly ours: keep the formulas framework-free and
|
|
5
|
-
// DB-free so they run anywhere (a Worker over D1 rows here, a page over Postgres rows there) and are
|
|
6
|
-
// unit-testable without a database. This module is client-safe and imports nothing from lib/server.
|
|
7
|
-
//
|
|
8
|
-
// The reading is a WINDOW aggregate, never a single scan. A grounded answer is non-deterministic
|
|
9
|
-
// (measured: 0.32-0.43 source-set overlap between two runs of one prompt on one engine within 24h), so
|
|
10
|
-
// a single draw per prompt is noise; the field pools repetition over a rolling window. That is the
|
|
11
|
-
// whole reason the storage is one-row-per-observation — see schema/geo.ts.
|
|
12
|
-
/** The field's replicated noise floor is roughly ±5–7pp. A point estimate is too imprecise to
|
|
13
|
-
* headline when its 95% CI has a half-width above 7pp (14pp full width). */
|
|
14
|
-
export const MAX_CI_HALF_WIDTH_PP = 7;
|
|
15
|
-
export const MAX_CI_WIDTH_PP = MAX_CI_HALF_WIDTH_PP * 2;
|
|
16
|
-
/** An intervention smaller than 7pp is not practically distinguishable from ordinary run-to-run
|
|
17
|
-
* movement even if a very large sample eventually makes it statistically significant. */
|
|
18
|
-
export const MIN_LIFT_PP = 7;
|
|
19
|
-
/** Familywise false-action budget for a frozen prompt-bank version. Repeated looks are handled by the
|
|
20
|
-
* e-process itself; this alpha is divided only across the fixed prompt × engine hypothesis family. */
|
|
21
|
-
export const PROMPT_ACTION_FWER = 0.05;
|
|
22
|
-
/** A fixed mixture of plausible loss rates. Each likelihood ratio is a test supermartingale for the
|
|
23
|
-
* composite null p>=50%; averaging them remains anytime-valid without choosing an effect after seeing
|
|
24
|
-
* the answers. This list is policy, so a change requires a prompt-bank-version reset. */
|
|
25
|
-
const PROMPT_GAP_ALTERNATIVES = [0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45];
|
|
26
|
-
/** Anytime-valid evidence that one prompt-engine mention rate is below 50%. The observation stream must
|
|
27
|
-
* be scoped to ONE frozen prompt-bank version and must include only complete scans. Repeated dashboard,
|
|
28
|
-
* MCP and cron looks spend no extra alpha; e-Bonferroni controls the fixed prompt × engine family under
|
|
29
|
-
* arbitrary dependence. Wilson intervals remain a separate display/precision rule. */
|
|
30
|
-
export function promptGapEvidence(brandMentions, observations, hypothesisCount, familywiseAlpha = PROMPT_ACTION_FWER) {
|
|
31
|
-
if (!Number.isSafeInteger(brandMentions) ||
|
|
32
|
-
!Number.isSafeInteger(observations) ||
|
|
33
|
-
brandMentions < 0 ||
|
|
34
|
-
observations < 0 ||
|
|
35
|
-
brandMentions > observations)
|
|
36
|
-
throw new RangeError("brandMentions and observations must be valid integer counts");
|
|
37
|
-
if (!Number.isSafeInteger(hypothesisCount) || hypothesisCount <= 0) {
|
|
38
|
-
throw new RangeError("hypothesisCount must be a positive safe integer");
|
|
39
|
-
}
|
|
40
|
-
if (!Number.isFinite(familywiseAlpha) || familywiseAlpha <= 0 || familywiseAlpha >= 1) {
|
|
41
|
-
throw new RangeError("familywiseAlpha must be between 0 and 1");
|
|
42
|
-
}
|
|
43
|
-
const failures = observations - brandMentions;
|
|
44
|
-
const logTerms = PROMPT_GAP_ALTERNATIVES.map((alternative) => brandMentions * Math.log(alternative / 0.5) +
|
|
45
|
-
failures * Math.log((1 - alternative) / 0.5));
|
|
46
|
-
const maxLog = Math.max(...logTerms);
|
|
47
|
-
const logEValue = maxLog +
|
|
48
|
-
Math.log(logTerms.reduce((total, value) => total + Math.exp(value - maxLog), 0) / PROMPT_GAP_ALTERNATIVES.length);
|
|
49
|
-
const threshold = hypothesisCount / familywiseAlpha;
|
|
50
|
-
return {
|
|
51
|
-
eValue: Math.exp(Math.min(logEValue, Math.log(Number.MAX_VALUE))),
|
|
52
|
-
threshold,
|
|
53
|
-
hypothesisCount,
|
|
54
|
-
familywiseAlpha,
|
|
55
|
-
proven: logEValue >= Math.log(threshold),
|
|
56
|
-
};
|
|
57
|
-
}
|
|
58
|
-
const pct = (num, den) => (den <= 0 ? null : Math.round((num / den) * 1000) / 10);
|
|
59
|
-
/** Unrounded Wilson bounds in percentage points. Contrast estimators consume these directly so
|
|
60
|
-
* intermediate rounding cannot move a final lift interval across zero. */
|
|
61
|
-
export function wilsonCiBounds(successes, total, z = 1.96) {
|
|
62
|
-
if (!Number.isFinite(successes) || !Number.isFinite(total) || !Number.isFinite(z) || total <= 0 || successes < 0 || successes > total || z <= 0)
|
|
63
|
-
return null;
|
|
64
|
-
const p = successes / total;
|
|
65
|
-
const z2 = z * z;
|
|
66
|
-
const denom = 1 + z2 / total;
|
|
67
|
-
const center = (p + z2 / (2 * total)) / denom;
|
|
68
|
-
const half = (z / denom) * Math.sqrt((p * (1 - p)) / total + z2 / (4 * total * total));
|
|
69
|
-
const clamp = (x) => Math.max(0, Math.min(1, x)) * 100;
|
|
70
|
-
return { lo: clamp(center - half), hi: clamp(center + half) };
|
|
71
|
-
}
|
|
72
|
-
/** 95% Wilson score interval for a binomial proportion `successes/total`, returned in PERCENT and
|
|
73
|
-
* rounded for display/storage. NULL when total=0. Wilson behaves correctly at small n and 0%/100%. */
|
|
74
|
-
export function wilsonCi(successes, total, z = 1.96) {
|
|
75
|
-
const ci = wilsonCiBounds(successes, total, z);
|
|
76
|
-
return ci ? { lo: Math.round(ci.lo * 10) / 10, hi: Math.round(ci.hi * 10) / 10 } : null;
|
|
77
|
-
}
|
|
78
|
-
/** Full width of an interval in pp; Infinity for a null (unmeasured) interval. */
|
|
79
|
-
export const ciWidth = (ci) => (ci ? ci.hi - ci.lo : Infinity);
|
|
80
|
-
/** The field threshold is about uncertainty around THE estimate, not half of an asymmetric interval's
|
|
81
|
-
* full width. Wilson intervals near 0%/100% are one-sided, so use the farther endpoint. */
|
|
82
|
-
export const ciMargin = (estimate, ci) => estimate === null || !ci ? Infinity : Math.max(estimate - ci.lo, ci.hi - estimate);
|
|
83
|
-
/** The one action gate for a prompt on one engine. The Wilson interval answers whether the estimate is
|
|
84
|
-
* precise enough to explain; the e-process answers whether repeated looks plus the fixed family still
|
|
85
|
-
* permit action. Callers must supply the FULL frozen prompt × engine family size, not the number of
|
|
86
|
-
* rows that happened to answer. */
|
|
87
|
-
export function evaluatePromptGap(brandMentions, observations, hypothesisCount, familywiseAlpha = PROMPT_ACTION_FWER) {
|
|
88
|
-
const evidence = promptGapEvidence(brandMentions, observations, hypothesisCount, familywiseAlpha);
|
|
89
|
-
const estimatePct = observations > 0 ? (brandMentions / observations) * 100 : null;
|
|
90
|
-
const bounds = wilsonCiBounds(brandMentions, observations);
|
|
91
|
-
const citedRateCi = bounds
|
|
92
|
-
? {
|
|
93
|
-
lo: Math.round(bounds.lo * 10) / 10,
|
|
94
|
-
hi: Math.round(bounds.hi * 10) / 10,
|
|
95
|
-
}
|
|
96
|
-
: null;
|
|
97
|
-
const citedRate = estimatePct === null ? null : estimatePct / 100;
|
|
98
|
-
const confidenceReady = estimatePct !== null &&
|
|
99
|
-
bounds !== null &&
|
|
100
|
-
bounds.hi < 50 &&
|
|
101
|
-
ciMargin(estimatePct, bounds) <= MAX_CI_HALF_WIDTH_PP;
|
|
102
|
-
if (estimatePct === null) {
|
|
103
|
-
return {
|
|
104
|
-
status: "unmeasured",
|
|
105
|
-
citedRate,
|
|
106
|
-
citedRateCi,
|
|
107
|
-
confidenceReady,
|
|
108
|
-
evidence,
|
|
109
|
-
reasons: ["no observations in the current prompt-bank version"],
|
|
110
|
-
};
|
|
111
|
-
}
|
|
112
|
-
if (estimatePct >= 50) {
|
|
113
|
-
return {
|
|
114
|
-
status: "not_gap",
|
|
115
|
-
citedRate,
|
|
116
|
-
citedRateCi,
|
|
117
|
-
confidenceReady: false,
|
|
118
|
-
evidence,
|
|
119
|
-
reasons: ["the brand is named in at least half of observations"],
|
|
120
|
-
};
|
|
121
|
-
}
|
|
122
|
-
const reasons = [];
|
|
123
|
-
if (!bounds || bounds.hi >= 50) {
|
|
124
|
-
reasons.push("the 95% Wilson interval does not lie wholly below 50%");
|
|
125
|
-
}
|
|
126
|
-
if (!bounds || ciMargin(estimatePct, bounds) > MAX_CI_HALF_WIDTH_PP) {
|
|
127
|
-
reasons.push(`the 95% interval extends more than ±${MAX_CI_HALF_WIDTH_PP}pp from the estimate`);
|
|
128
|
-
}
|
|
129
|
-
if (!evidence.proven) {
|
|
130
|
-
reasons.push(`the anytime-valid familywise e-value ${evidence.eValue.toPrecision(3)} is below ${evidence.threshold.toFixed(0)}`);
|
|
131
|
-
}
|
|
132
|
-
return {
|
|
133
|
-
status: confidenceReady && evidence.proven ? "proven_gap" : "research_only",
|
|
134
|
-
citedRate,
|
|
135
|
-
citedRateCi,
|
|
136
|
-
confidenceReady,
|
|
137
|
-
evidence,
|
|
138
|
-
reasons,
|
|
139
|
-
};
|
|
140
|
-
}
|
|
141
|
-
/** Deterministic cluster bootstrap for share of voice. One answer can name several rivals, so its SoV
|
|
142
|
-
* contributions are not Bernoulli trials and a Wilson interval over raw brand+rival mentions would
|
|
143
|
-
* be falsely narrow. Resampling whole observations preserves that within-answer clustering. A
|
|
144
|
-
* cluster-effective Wilson floor prevents the percentile bootstrap's known zero-width failure when
|
|
145
|
-
* a small sample lands entirely at 0% or 100%. */
|
|
146
|
-
export function bootstrapSovCi(observations, samples = 10_000) {
|
|
147
|
-
const contributions = observations
|
|
148
|
-
.map((o) => ({ brand: o.brandMentioned ? 1 : 0, rivals: o.competitors.length }))
|
|
149
|
-
.sort((a, b) => a.brand - b.brand || a.rivals - b.rivals);
|
|
150
|
-
const brand = contributions.reduce((total, x) => total + x.brand, 0);
|
|
151
|
-
const mentionWeights = contributions.map((x) => x.brand + x.rivals);
|
|
152
|
-
const mentions = mentionWeights.reduce((total, weight) => total + weight, 0);
|
|
153
|
-
if (mentions === 0 || samples <= 0)
|
|
154
|
-
return null;
|
|
155
|
-
let state = 0x9e3779b9;
|
|
156
|
-
const shares = [];
|
|
157
|
-
for (let sample = 0; sample < samples; sample++) {
|
|
158
|
-
let sampleBrand = 0;
|
|
159
|
-
let sampleRivals = 0;
|
|
160
|
-
for (let draw = 0; draw < contributions.length; draw++) {
|
|
161
|
-
state ^= state << 13;
|
|
162
|
-
state ^= state >>> 17;
|
|
163
|
-
state ^= state << 5;
|
|
164
|
-
const picked = contributions[(state >>> 0) % contributions.length];
|
|
165
|
-
sampleBrand += picked.brand;
|
|
166
|
-
sampleRivals += picked.rivals;
|
|
167
|
-
}
|
|
168
|
-
if (sampleBrand + sampleRivals > 0)
|
|
169
|
-
shares.push((sampleBrand / (sampleBrand + sampleRivals)) * 100);
|
|
170
|
-
}
|
|
171
|
-
if (!shares.length)
|
|
172
|
-
return null;
|
|
173
|
-
shares.sort((a, b) => a - b);
|
|
174
|
-
const quantile = (q) => shares[Math.min(shares.length - 1, Math.max(0, Math.floor(q * shares.length)))];
|
|
175
|
-
const bootstrap = { lo: quantile(0.025), hi: quantile(0.975) };
|
|
176
|
-
// Kish effective sample size treats each answer as one cluster and discounts answers that contribute
|
|
177
|
-
// many rival mentions. Fractional Wilson successes are the SoV estimate at that effective n.
|
|
178
|
-
const squaredWeights = mentionWeights.reduce((total, weight) => total + weight * weight, 0);
|
|
179
|
-
const effectiveN = (mentions * mentions) / squaredWeights;
|
|
180
|
-
const floor = wilsonCi((brand / mentions) * effectiveN, effectiveN);
|
|
181
|
-
return {
|
|
182
|
-
lo: Math.round(Math.min(bootstrap.lo, floor.lo) * 10) / 10,
|
|
183
|
-
hi: Math.round(Math.max(bootstrap.hi, floor.hi) * 10) / 10,
|
|
184
|
-
};
|
|
185
|
-
}
|
|
186
|
-
/** Aggregate a window of observations into one summary. Pure. Elmo's `computeReportMetrics` shape:
|
|
187
|
-
* share = brand / (brand + competitors), presence = brand / total, sourced = sourced / grounded, each
|
|
188
|
-
* with an explicit empty-denominator → null (never a divide, never a misleading 0). */
|
|
189
|
-
export function computeGeoWindow(observations) {
|
|
190
|
-
let brandPresences = 0;
|
|
191
|
-
let competitorPresences = 0;
|
|
192
|
-
let groundedObservations = 0;
|
|
193
|
-
let sourcedObservations = 0;
|
|
194
|
-
const competitorsSeen = new Set();
|
|
195
|
-
const competitorCounts = new Map();
|
|
196
|
-
for (const o of observations) {
|
|
197
|
-
if (o.brandMentioned)
|
|
198
|
-
brandPresences += 1;
|
|
199
|
-
for (const label of o.competitors) {
|
|
200
|
-
competitorPresences += 1;
|
|
201
|
-
competitorsSeen.add(label.toLowerCase());
|
|
202
|
-
competitorCounts.set(label, (competitorCounts.get(label) ?? 0) + 1);
|
|
203
|
-
}
|
|
204
|
-
if (o.grounded) {
|
|
205
|
-
groundedObservations += 1;
|
|
206
|
-
if (o.sourced === true)
|
|
207
|
-
sourcedObservations += 1;
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
const total = observations.length;
|
|
211
|
-
const sovPct = pct(brandPresences, brandPresences + competitorPresences);
|
|
212
|
-
const presenceRatePct = pct(brandPresences, total);
|
|
213
|
-
const sourcedRatePct = pct(sourcedObservations, groundedObservations);
|
|
214
|
-
const sovCi = bootstrapSovCi(observations);
|
|
215
|
-
const presenceRateCi = wilsonCi(brandPresences, total);
|
|
216
|
-
const sourcedRateCi = wilsonCi(sourcedObservations, groundedObservations);
|
|
217
|
-
return {
|
|
218
|
-
sovPct,
|
|
219
|
-
presenceRatePct,
|
|
220
|
-
sourcedRatePct,
|
|
221
|
-
denominatorWidth: competitorsSeen.size + (brandPresences > 0 ? 1 : 0),
|
|
222
|
-
observations: total,
|
|
223
|
-
groundedObservations,
|
|
224
|
-
competitors: [...competitorCounts.entries()]
|
|
225
|
-
.map(([label, count]) => ({ label, count }))
|
|
226
|
-
.sort((a, b) => b.count - a.count || a.label.localeCompare(b.label)),
|
|
227
|
-
sovCi,
|
|
228
|
-
presenceRateCi,
|
|
229
|
-
sourcedRateCi,
|
|
230
|
-
// A window with NO observations makes every metric null, so every `x !== null` test is false and
|
|
231
|
-
// the OR was FALSE — an engine that measured nothing reported as high-confidence, which the header
|
|
232
|
-
// then persisted and the MCP scoreboard rendered as "every metric is within ±7pp". Nothing measured
|
|
233
|
-
// is the least confident state there is.
|
|
234
|
-
lowConfidence: total === 0 ||
|
|
235
|
-
(sovPct !== null && ciMargin(sovPct, sovCi) > MAX_CI_HALF_WIDTH_PP) ||
|
|
236
|
-
(presenceRatePct !== null && ciMargin(presenceRatePct, presenceRateCi) > MAX_CI_HALF_WIDTH_PP) ||
|
|
237
|
-
(sourcedRatePct !== null && ciMargin(sourcedRatePct, sourcedRateCi) > MAX_CI_HALF_WIDTH_PP),
|
|
238
|
-
};
|
|
239
|
-
}
|
|
240
|
-
export function computeGeoPromptWindows(observations) {
|
|
241
|
-
const byPrompt = new Map();
|
|
242
|
-
for (const o of observations) {
|
|
243
|
-
const arr = byPrompt.get(o.promptText);
|
|
244
|
-
if (arr)
|
|
245
|
-
arr.push(o);
|
|
246
|
-
else
|
|
247
|
-
byPrompt.set(o.promptText, [o]);
|
|
248
|
-
}
|
|
249
|
-
const out = [];
|
|
250
|
-
for (const [promptText, obs] of byPrompt) {
|
|
251
|
-
const brandMentions = obs.filter((o) => o.brandMentioned).length;
|
|
252
|
-
const counts = new Map();
|
|
253
|
-
for (const o of obs)
|
|
254
|
-
for (const c of o.competitors)
|
|
255
|
-
counts.set(c, (counts.get(c) ?? 0) + 1);
|
|
256
|
-
const citedRateCiPct = wilsonCi(brandMentions, obs.length) ?? { lo: 0, hi: 100 };
|
|
257
|
-
const citedRate = obs.length > 0 ? brandMentions / obs.length : 0;
|
|
258
|
-
const crossesDecisionBoundary = citedRate < 0.5 ? citedRateCiPct.hi >= 50 : citedRateCiPct.lo < 50;
|
|
259
|
-
out.push({
|
|
260
|
-
promptText,
|
|
261
|
-
observations: obs.length,
|
|
262
|
-
brandMentions,
|
|
263
|
-
citedRate,
|
|
264
|
-
citedRateCi: {
|
|
265
|
-
lo: Math.round(citedRateCiPct.lo * 10) / 1000,
|
|
266
|
-
hi: Math.round(citedRateCiPct.hi * 10) / 1000,
|
|
267
|
-
},
|
|
268
|
-
lowConfidence: ciMargin(citedRate * 100, citedRateCiPct) > MAX_CI_HALF_WIDTH_PP || crossesDecisionBoundary,
|
|
269
|
-
competitors: [...counts.entries()].map(([label, count]) => ({ label, count })).sort((a, b) => b.count - a.count),
|
|
270
|
-
});
|
|
271
|
-
}
|
|
272
|
-
return out.sort((a, b) => a.citedRate - b.citedRate || b.competitors.length - a.competitors.length);
|
|
273
|
-
}
|
package/dist/runner.d.ts
DELETED
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
import { type BrandConfig } from "./matchers.ts";
|
|
2
|
-
import type { GeoObservation } from "./metrics.ts";
|
|
3
|
-
/** One measured row plus the non-metric facts a consumer usually stores beside it. */
|
|
4
|
-
export type GeoScanRow = GeoObservation & {
|
|
5
|
-
citedHosts: string[];
|
|
6
|
-
searchQueries: number;
|
|
7
|
-
};
|
|
8
|
-
export type GeoScanConfig = {
|
|
9
|
-
brand: BrandConfig;
|
|
10
|
-
competitors: readonly {
|
|
11
|
-
domain: string;
|
|
12
|
-
label: string;
|
|
13
|
-
}[];
|
|
14
|
-
prompts: readonly string[];
|
|
15
|
-
/** The row's engine label, e.g. "gemini-grounded". The aggregator groups by it and never averages
|
|
16
|
-
* across engines — they diverge wildly. */
|
|
17
|
-
engine: string;
|
|
18
|
-
/** Whether `ask` runs with retrieval. Controls both the system nudge and whether grounding metadata
|
|
19
|
-
* is read. An ungrounded scan's `sourced` is always null (unmeasured, never false). */
|
|
20
|
-
grounded: boolean;
|
|
21
|
-
/** Answers per prompt per scan. Default 1 — a rolling window across scans pools better than a burst
|
|
22
|
-
* on one day (portfolio-level, measured by the field); raise only alongside a cadence change. */
|
|
23
|
-
repetitions?: number;
|
|
24
|
-
/** Override the system prompt. The default asks for concrete app/product recommendations and, when
|
|
25
|
-
* grounded, to search first. */
|
|
26
|
-
system?: {
|
|
27
|
-
grounded?: string;
|
|
28
|
-
ungrounded?: string;
|
|
29
|
-
};
|
|
30
|
-
};
|
|
31
|
-
export type GeoScanPorts = {
|
|
32
|
-
/** One LLM call. Throwing loses this observation only. */
|
|
33
|
-
ask(prompt: string, system: string): Promise<{
|
|
34
|
-
text?: string | null;
|
|
35
|
-
providerMetadata?: unknown;
|
|
36
|
-
}>;
|
|
37
|
-
/** Persist one row as it is measured. Catch your own storage errors — a throw here also costs only
|
|
38
|
-
* this observation. */
|
|
39
|
-
onObservation?(row: GeoScanRow): void | Promise<void>;
|
|
40
|
-
/** Polled before every call; true stops the scan with a stoppedReason. */
|
|
41
|
-
budget?: {
|
|
42
|
-
overBudget(): boolean;
|
|
43
|
-
};
|
|
44
|
-
};
|
|
45
|
-
export type GeoScanResult = {
|
|
46
|
-
observations: GeoScanRow[];
|
|
47
|
-
/** Raw answer texts, for a consumer's own mining pass. Never persisted by the kit. */
|
|
48
|
-
answers: {
|
|
49
|
-
prompt: string;
|
|
50
|
-
answer: string;
|
|
51
|
-
}[];
|
|
52
|
-
/** Retrieved hosts per prompt (accumulated across repetitions) — venue analysis reads this. */
|
|
53
|
-
hostsByPrompt: Map<string, string[]>;
|
|
54
|
-
/** Billable search queries across the scan — the only honest bound on a grounded bill. */
|
|
55
|
-
searchQueries: number;
|
|
56
|
-
stoppedReason: string | null;
|
|
57
|
-
};
|
|
58
|
-
export declare function runGeoScan(config: GeoScanConfig, ports: GeoScanPorts): Promise<GeoScanResult>;
|
package/dist/runner.js
DELETED
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
// geovouch/runner — the generic scan loop, with every side effect injected as a port. The kit stays
|
|
2
|
-
// IO-free by construction: it never imports an LLM client, a database or a clock — the consumer hands
|
|
3
|
-
// in `ask` (one LLM call) and optionally `onObservation` (persist one row) and `budget` (spend guard),
|
|
4
|
-
// and the runner owns only the loop's SHAPE: repetition, fail-soft per observation, grounded-answer
|
|
5
|
-
// parsing, brand/competitor detection.
|
|
6
|
-
//
|
|
7
|
-
// This shape is lifted from the first consumer's production loop, where it ran daily against grounded
|
|
8
|
-
// Gemini. What each port is for, learned there:
|
|
9
|
-
// * `ask` returns the raw text + providerMetadata; a throw inside it loses ONE observation, never
|
|
10
|
-
// the scan (no fake data is ever recorded for a failed call).
|
|
11
|
-
// * `budget.overBudget()` is polled before every call, because a grounded prompt's search fan-out is
|
|
12
|
-
// the model's choice, not the caller's — the bank size does not bound the bill; only counting the
|
|
13
|
-
// answers does (see gemini.ts).
|
|
14
|
-
// * `onObservation` receives each row as it is measured, so storage is append-per-row and a failed
|
|
15
|
-
// insert (the port's own catch) costs one row, never the scan.
|
|
16
|
-
import { buildBrandMatcher, buildCompetitorMatchers, namesIn } from "./matchers.js";
|
|
17
|
-
import { groundedHosts, groundedQueryCount } from "./gemini.js";
|
|
18
|
-
const DEFAULT_GROUNDED_SYSTEM = "You are a helpful assistant recommending apps. SEARCH THE WEB FIRST, then answer naturally and concretely, naming specific apps you'd recommend based on what you found.";
|
|
19
|
-
const DEFAULT_UNGROUNDED_SYSTEM = "You are a helpful assistant recommending apps. Answer naturally and concretely, naming specific apps you'd recommend.";
|
|
20
|
-
export async function runGeoScan(config, ports) {
|
|
21
|
-
const brand = buildBrandMatcher(config.brand);
|
|
22
|
-
const matchers = buildCompetitorMatchers(config.competitors);
|
|
23
|
-
const repetitions = config.repetitions ?? 1;
|
|
24
|
-
const system = config.grounded
|
|
25
|
-
? (config.system?.grounded ?? DEFAULT_GROUNDED_SYSTEM)
|
|
26
|
-
: (config.system?.ungrounded ?? DEFAULT_UNGROUNDED_SYSTEM);
|
|
27
|
-
const observations = [];
|
|
28
|
-
const answers = [];
|
|
29
|
-
const hostsByPrompt = new Map();
|
|
30
|
-
let searchQueries = 0;
|
|
31
|
-
let stoppedReason = null;
|
|
32
|
-
for (const prompt of config.prompts) {
|
|
33
|
-
for (let rep = 0; rep < repetitions; rep++) {
|
|
34
|
-
if (ports.budget?.overBudget()) {
|
|
35
|
-
stoppedReason = `budget reached after ${searchQueries} search queries`;
|
|
36
|
-
break;
|
|
37
|
-
}
|
|
38
|
-
let answer = "";
|
|
39
|
-
let hosts = [];
|
|
40
|
-
let queries = 0;
|
|
41
|
-
try {
|
|
42
|
-
const { text, providerMetadata } = await ports.ask(prompt, system);
|
|
43
|
-
answer = text ?? "";
|
|
44
|
-
hosts = config.grounded ? groundedHosts(providerMetadata) : [];
|
|
45
|
-
queries = config.grounded ? groundedQueryCount(providerMetadata) : 0;
|
|
46
|
-
searchQueries += queries;
|
|
47
|
-
}
|
|
48
|
-
catch {
|
|
49
|
-
continue; // this observation just doesn't contribute (no fake data)
|
|
50
|
-
}
|
|
51
|
-
// Grounding is per-ANSWER: a memory answer inside a grounded scan retrieved nothing, so its
|
|
52
|
-
// sourced is null (unmeasured), never false.
|
|
53
|
-
const grounded = queries > 0 || hosts.length > 0;
|
|
54
|
-
const row = {
|
|
55
|
-
promptText: prompt,
|
|
56
|
-
engine: config.engine,
|
|
57
|
-
grounded,
|
|
58
|
-
brandMentioned: brand.mentioned(answer),
|
|
59
|
-
sourced: grounded ? hosts.some((h) => brand.sourcedHost(h)) : null,
|
|
60
|
-
competitors: namesIn(answer, matchers),
|
|
61
|
-
citedHosts: hosts,
|
|
62
|
-
searchQueries: queries,
|
|
63
|
-
};
|
|
64
|
-
observations.push(row);
|
|
65
|
-
if (answer)
|
|
66
|
-
answers.push({ prompt, answer });
|
|
67
|
-
if (hosts.length)
|
|
68
|
-
hostsByPrompt.set(prompt, [...(hostsByPrompt.get(prompt) ?? []), ...hosts]);
|
|
69
|
-
try {
|
|
70
|
-
await ports.onObservation?.(row);
|
|
71
|
-
}
|
|
72
|
-
catch {
|
|
73
|
-
/* a failed persist costs one row, never the scan */
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
if (stoppedReason)
|
|
77
|
-
break;
|
|
78
|
-
}
|
|
79
|
-
return { observations, answers, hostsByPrompt, searchQueries, stoppedReason };
|
|
80
|
-
}
|