geovouch 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +121 -0
- package/dist/gemini.d.ts +28 -0
- package/dist/gemini.js +62 -0
- package/dist/matchers.d.ts +32 -0
- package/dist/matchers.js +46 -0
- package/dist/metrics.d.ts +139 -0
- package/dist/metrics.js +273 -0
- package/dist/runner.d.ts +58 -0
- package/dist/runner.js +80 -0
- package/package.json +37 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 The Influence Company
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# geovouch
|
|
2
|
+
|
|
3
|
+
Measure whether AI assistants name your brand when buyers ask them questions — and know when that
|
|
4
|
+
measurement is strong enough to act on.
|
|
5
|
+
|
|
6
|
+
Pure TypeScript. **Zero dependencies, zero IO.** Runs in a Cloudflare Worker, in Node, in a browser,
|
|
7
|
+
in a test with no database.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm i geovouch
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## The one idea
|
|
16
|
+
|
|
17
|
+
**One row per (prompt × engine × repetition). Never a metric.**
|
|
18
|
+
|
|
19
|
+
A grounded AI answer is non-deterministic. Two runs of the *same* prompt on the *same* engine within
|
|
20
|
+
24 hours share only 0.32–0.43 of their source set (St. Gallen, arXiv:2604.07585). So a single reading
|
|
21
|
+
per prompt is a coin flip, and a weekly dashboard number computed from one draw is noise with a
|
|
22
|
+
decimal point.
|
|
23
|
+
|
|
24
|
+
Everything here computes over a pooled window of atomic observations, and **never averages across
|
|
25
|
+
engines** — they diverge by ~46×, so one blended "AI visibility score" is an average of four different
|
|
26
|
+
instruments.
|
|
27
|
+
|
|
28
|
+
```ts
|
|
29
|
+
import { computeGeoWindow, type GeoObservation } from "geovouch/metrics";
|
|
30
|
+
|
|
31
|
+
const observations: GeoObservation[] = [
|
|
32
|
+
{ promptText: "best AI notetaker?", engine: "gemini", grounded: true,
|
|
33
|
+
brandMentioned: true, sourced: false, competitors: ["Otter"] },
|
|
34
|
+
// …one row per answer, pooled over your window
|
|
35
|
+
];
|
|
36
|
+
|
|
37
|
+
const w = computeGeoWindow(observations);
|
|
38
|
+
w.sovPct; // share of voice %, or null when nothing was named at all
|
|
39
|
+
w.sovCi; // { lo, hi } in pp — an observation-CLUSTER bootstrap, because one answer
|
|
40
|
+
// contributes several brand presences and they are not independent draws
|
|
41
|
+
w.presenceRateCi; // Wilson score interval — this one IS a Bernoulli rate
|
|
42
|
+
w.sourcedRateCi; // Wilson, over the GROUNDED observations only
|
|
43
|
+
w.lowConfidence; // true while any measured metric is wider than ±7pp
|
|
44
|
+
w.denominatorWidth; // distinct brands in the denominator — losing to 2 rivals ≠ losing to 20
|
|
45
|
+
w.observations; // the sample size. This IS the number that makes the rest readable.
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
`sovPct === null` means **nothing was named**, which is not a zero share. That distinction is load
|
|
49
|
+
bearing throughout: *unmeasured is never a zero.*
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
## Knowing when you may act
|
|
54
|
+
|
|
55
|
+
A dashboard number and a number you may spend money on are different things. The gate is explicit:
|
|
56
|
+
|
|
57
|
+
```ts
|
|
58
|
+
import { evaluatePromptGap, PROMPT_ACTION_FWER } from "geovouch/metrics";
|
|
59
|
+
|
|
60
|
+
// 10 prompts × 4 engines = 40 hypotheses in the frozen family
|
|
61
|
+
const decision = evaluatePromptGap(brandMentions, observations, 40);
|
|
62
|
+
|
|
63
|
+
decision.status; // "unmeasured" | "not_gap" | "research_only" | "proven_gap"
|
|
64
|
+
decision.reasons; // why it is not proven yet, in sentences
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Three independent conditions, and all of them must hold:
|
|
68
|
+
|
|
69
|
+
| condition | what it rules out |
|
|
70
|
+
|---|---|
|
|
71
|
+
| Wilson 95% interval, half-width ≤ 7pp | a point estimate that a few more draws would move |
|
|
72
|
+
| interval entirely below 50% | "we might be losing this question" |
|
|
73
|
+
| anytime-valid e-value ≥ family size ÷ α | the 40-hypotheses-one-looks-significant problem |
|
|
74
|
+
|
|
75
|
+
The third is the one most tools skip. It is a **one-sided finite-mixture Bernoulli e-process** against
|
|
76
|
+
the composite null *p ≥ 0.5*, so **repeated daily looks cost no alpha** (Ville's inequality) — you can
|
|
77
|
+
check every morning without inflating your error rate, which is exactly what a dashboard invites you
|
|
78
|
+
to do. The threshold is e-Bonferroni: `familySize / α`, so 40 hypotheses at α=0.05 needs an e-value of
|
|
79
|
+
**800**.
|
|
80
|
+
|
|
81
|
+
At n≈216 per cell, roughly a third of genuinely-low prompts clear it. That is the honest yield, and a
|
|
82
|
+
tool that reports more than that is reporting noise.
|
|
83
|
+
|
|
84
|
+
> A bank change is a **ruler** change. Bump your prompt-bank version when the bank changes, or the
|
|
85
|
+
> frozen hypothesis family silently grows past the denominator it is priced with.
|
|
86
|
+
|
|
87
|
+
---
|
|
88
|
+
|
|
89
|
+
## The rest of the surface
|
|
90
|
+
|
|
91
|
+
```ts
|
|
92
|
+
import { buildBrandMatcher, buildCompetitorMatchers, namesIn } from "geovouch/matchers";
|
|
93
|
+
import { groundedHosts, groundedQueryCount } from "geovouch/gemini";
|
|
94
|
+
import { runGeoScan } from "geovouch/runner";
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
- **`geovouch/matchers`** — brand and competitor detection over answer prose. Handles the spaced form
|
|
98
|
+
of a CamelCase brand, which is how 93% of them actually appear.
|
|
99
|
+
- **`geovouch/gemini`** — parse Gemini grounding metadata into retrieved hosts and billable query
|
|
100
|
+
counts. A host must be domain-shaped; a redirector is not a publisher.
|
|
101
|
+
- **`geovouch/runner`** — the scan loop, with every engine and store injected as a **port**. No IO of
|
|
102
|
+
its own, so it is testable without a network.
|
|
103
|
+
|
|
104
|
+
Storage, Cloudflare D1 wiring and the multi-engine Apify sweep live in sibling packages
|
|
105
|
+
(`geovouch-store`, `geovouch-engines`). This package is the part with no opinions about where your
|
|
106
|
+
data lives.
|
|
107
|
+
|
|
108
|
+
---
|
|
109
|
+
|
|
110
|
+
## What this measures, and what it does not
|
|
111
|
+
|
|
112
|
+
`brandMentioned` is a regex over prose — a model can satisfy it from memory, or from a confabulation.
|
|
113
|
+
`sourced` asks whether one of *your* domains was among the URLs the engine actually **retrieved**, and
|
|
114
|
+
it is the only signal a model cannot produce from memory. It is `null`, never `false`, when an answer
|
|
115
|
+
did not ground: an engine that answered without searching has told you nothing about your retrieval.
|
|
116
|
+
|
|
117
|
+
This library measures. It does not act, and it does not publish.
|
|
118
|
+
|
|
119
|
+
---
|
|
120
|
+
|
|
121
|
+
MIT © The Influence Company
|
package/dist/gemini.d.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/** Google's opaque grounding redirect. EVERY `web.uri` is one of these, so it is never a source — a
|
|
2
|
+
* set containing only this host means "we read the wrong field", not "Google cited itself". */
|
|
3
|
+
export declare const GROUNDING_REDIRECT_HOST = "vertexaisearch.cloud.google.com";
|
|
4
|
+
/** Hostnames the engine retrieved, off Gemini's grounding metadata. Empty when the answer did not
|
|
5
|
+
* ground.
|
|
6
|
+
*
|
|
7
|
+
* READ `web.title`, NOT `web.uri`, AND NEVER `web.domain`. Measured: every chunk is
|
|
8
|
+
* `{web:{uri,title}}`, `web.domain` is sent ZERO times — and @ai-sdk/google declares
|
|
9
|
+
* `web: z.object({uri, title})`, so zod would strip a future `domain` before it reached us; a
|
|
10
|
+
* fallback to it would read as a safety net while being unreachable code. Every `uri` is the opaque
|
|
11
|
+
* redirect above. The publisher is in `title`, which on the Developer API is a bare domain — but not
|
|
12
|
+
* contractually (Vertex puts page titles there), so a title is accepted only when it LOOKS like a
|
|
13
|
+
* hostname; "Best AI apps | Some Site" is dropped rather than URL-parsed into garbage. */
|
|
14
|
+
export declare function groundedHosts(providerMetadata: unknown): string[];
|
|
15
|
+
/** Search queries the grounded answer actually executed — the unit Gemini 3.x BILLS on. The model
|
|
16
|
+
* decides the fan-out per prompt (may be zero: a memory answer inside a grounded request), so a
|
|
17
|
+
* prompt bank of a known size is NOT a bill of a known size; counting the answer is the only bound. */
|
|
18
|
+
export declare function groundedQueryCount(providerMetadata: unknown): number;
|
|
19
|
+
/** $14 per 1,000 search queries — ai.google.dev/gemini-api/docs/pricing, read 2026-09-01. The 2.5
|
|
20
|
+
* line billed per grounded PROMPT; the 3.x line bills per SEARCH QUERY. The 5,000-free-per-month
|
|
21
|
+
* allowance is deliberately NOT modelled: it is shared across every Gemini caller on an account, so
|
|
22
|
+
* a second grounded call site would silently make the number wrong. Charging from the first query is
|
|
23
|
+
* conservative and cannot drift. */
|
|
24
|
+
export declare const GROUNDED_QUERY_CENTS = 1.4;
|
|
25
|
+
/** What a run's grounded searches cost, in USD. The rate lives here and nowhere else — a console that
|
|
26
|
+
* multiplied the query count by its own copy of the price would go quietly wrong the day Google
|
|
27
|
+
* moves it. */
|
|
28
|
+
export declare const groundedCostUsd: (searchQueries: number) => number;
|
package/dist/gemini.js
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
// geovouch/gemini — grounding-metadata parsing for the Gemini Developer API. Product-agnostic: see
|
|
2
|
+
// BOUNDARY.md. Every claim here was measured against the live API on 2026-09-04 (49 grounding chunks
|
|
3
|
+
// across three calls through `ai` + `@ai-sdk/google`), because the first consumer shipped two months
|
|
4
|
+
// of dead data by trusting a field the API never sends.
|
|
5
|
+
/** Google's opaque grounding redirect. EVERY `web.uri` is one of these, so it is never a source — a
|
|
6
|
+
* set containing only this host means "we read the wrong field", not "Google cited itself". */
|
|
7
|
+
export const GROUNDING_REDIRECT_HOST = "vertexaisearch.cloud.google.com";
|
|
8
|
+
/** Hostnames the engine retrieved, off Gemini's grounding metadata. Empty when the answer did not
|
|
9
|
+
* ground.
|
|
10
|
+
*
|
|
11
|
+
* READ `web.title`, NOT `web.uri`, AND NEVER `web.domain`. Measured: every chunk is
|
|
12
|
+
* `{web:{uri,title}}`, `web.domain` is sent ZERO times — and @ai-sdk/google declares
|
|
13
|
+
* `web: z.object({uri, title})`, so zod would strip a future `domain` before it reached us; a
|
|
14
|
+
* fallback to it would read as a safety net while being unreachable code. Every `uri` is the opaque
|
|
15
|
+
* redirect above. The publisher is in `title`, which on the Developer API is a bare domain — but not
|
|
16
|
+
* contractually (Vertex puts page titles there), so a title is accepted only when it LOOKS like a
|
|
17
|
+
* hostname; "Best AI apps | Some Site" is dropped rather than URL-parsed into garbage. */
|
|
18
|
+
export function groundedHosts(providerMetadata) {
|
|
19
|
+
const meta = providerMetadata;
|
|
20
|
+
const chunks = meta?.google?.groundingMetadata?.groundingChunks ?? [];
|
|
21
|
+
const hosts = new Set();
|
|
22
|
+
for (const c of chunks) {
|
|
23
|
+
const host = hostFromTitle(c?.web?.title) ?? hostFromUri(c?.web?.uri);
|
|
24
|
+
if (host && host !== GROUNDING_REDIRECT_HOST)
|
|
25
|
+
hosts.add(host);
|
|
26
|
+
}
|
|
27
|
+
return [...hosts];
|
|
28
|
+
}
|
|
29
|
+
/** A bare hostname (`example.ai`, `www.reddit.com`) → normalized; anything else → null. */
|
|
30
|
+
function hostFromTitle(title) {
|
|
31
|
+
const t = title?.trim().toLowerCase();
|
|
32
|
+
if (!t || !/^[a-z0-9-]+(\.[a-z0-9-]+)+$/.test(t))
|
|
33
|
+
return null;
|
|
34
|
+
return t.replace(/^www\./, "");
|
|
35
|
+
}
|
|
36
|
+
function hostFromUri(uri) {
|
|
37
|
+
if (!uri)
|
|
38
|
+
return null;
|
|
39
|
+
try {
|
|
40
|
+
return new URL(uri.startsWith("http") ? uri : `https://${uri}`).hostname.replace(/^www\./, "");
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
/** Search queries the grounded answer actually executed — the unit Gemini 3.x BILLS on. The model
|
|
47
|
+
* decides the fan-out per prompt (may be zero: a memory answer inside a grounded request), so a
|
|
48
|
+
* prompt bank of a known size is NOT a bill of a known size; counting the answer is the only bound. */
|
|
49
|
+
export function groundedQueryCount(providerMetadata) {
|
|
50
|
+
const meta = providerMetadata;
|
|
51
|
+
return meta?.google?.groundingMetadata?.webSearchQueries?.length ?? 0;
|
|
52
|
+
}
|
|
53
|
+
/** $14 per 1,000 search queries — ai.google.dev/gemini-api/docs/pricing, read 2026-09-01. The 2.5
|
|
54
|
+
* line billed per grounded PROMPT; the 3.x line bills per SEARCH QUERY. The 5,000-free-per-month
|
|
55
|
+
* allowance is deliberately NOT modelled: it is shared across every Gemini caller on an account, so
|
|
56
|
+
* a second grounded call site would silently make the number wrong. Charging from the first query is
|
|
57
|
+
* conservative and cannot drift. */
|
|
58
|
+
export const GROUNDED_QUERY_CENTS = 1.4;
|
|
59
|
+
/** What a run's grounded searches cost, in USD. The rate lives here and nowhere else — a console that
|
|
60
|
+
* multiplied the query count by its own copy of the price would go quietly wrong the day Google
|
|
61
|
+
* moves it. */
|
|
62
|
+
export const groundedCostUsd = (searchQueries) => Math.round(searchQueries * GROUNDED_QUERY_CENTS) / 100;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export type BrandConfig = {
|
|
2
|
+
/** The brand word as engines write it, e.g. "Acme". Matched word-bounded, case-insensitive. */
|
|
3
|
+
name: string;
|
|
4
|
+
/** Registrable domains that count as the brand's own pages among retrieved sources ("acme.ai"
|
|
5
|
+
* matches itself and any subdomain). */
|
|
6
|
+
domains: readonly string[];
|
|
7
|
+
/** Extra spellings that also count as a mention (a common misspelling, a former name). */
|
|
8
|
+
aliases?: readonly string[];
|
|
9
|
+
};
|
|
10
|
+
export type BrandMatcher = {
|
|
11
|
+
/** Was the brand NAMED in the prose? A model can satisfy this from memory or confabulation — pair
|
|
12
|
+
* it with `sourcedHost` on retrieved sources, which it cannot fake. */
|
|
13
|
+
mentioned(answer: string): boolean;
|
|
14
|
+
/** Is this retrieved host one of ours? */
|
|
15
|
+
sourcedHost(host: string): boolean;
|
|
16
|
+
};
|
|
17
|
+
export declare function buildBrandMatcher(config: BrandConfig): BrandMatcher;
|
|
18
|
+
export type CompetitorMatcher = {
|
|
19
|
+
label: string;
|
|
20
|
+
re: RegExp;
|
|
21
|
+
};
|
|
22
|
+
/** Build presence matchers for a competitor set. Tokenizes labels at BOTH kinds of boundary — the
|
|
23
|
+
* separators an engine may drop, and the CamelCase seam an all-together label hides — then rejoins
|
|
24
|
+
* with an optional separator: "AcmeAI" → /\bAcme[.\s-]?AI\b/i, matching "Acme AI", "AcmeAI" and
|
|
25
|
+
* "Acme.AI" alike. The FULL domain is the second alternative, never its root. */
|
|
26
|
+
export declare function buildCompetitorMatchers(competitors: readonly {
|
|
27
|
+
domain: string;
|
|
28
|
+
label: string;
|
|
29
|
+
}[]): CompetitorMatcher[];
|
|
30
|
+
/** Competitor labels named in one answer — the atoms of share-of-voice's denominator. A false
|
|
31
|
+
* positive here is indistinguishable from the measured brand losing ground. */
|
|
32
|
+
export declare function namesIn(answer: string, matchers: readonly CompetitorMatcher[]): string[];
|
package/dist/matchers.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// geovouch/matchers — brand + competitor presence detection over engine answer text. Product-agnostic:
|
|
2
|
+
// see BOUNDARY.md. Every rule here was paid for by a measured incident on the first consumer, so the
|
|
3
|
+
// rules travel with the code:
|
|
4
|
+
// * A competitor is matched by LABEL (separator- AND CamelCase-flexible) or FULL DOMAIN — never a
|
|
5
|
+
// bare domain root. A rival on a `<noun>.ai` domain whose root is an everyday word the measured
|
|
6
|
+
// product's own copy uses constantly once matched as `\b<noun>\b` and scored a presence on 8/10
|
|
7
|
+
// answers against a true 5/10 — and every false positive is a point in share-of-voice's
|
|
8
|
+
// DENOMINATOR, i.e. it pushes the measured brand DOWN.
|
|
9
|
+
// * CamelCase must tokenize, because engines write the spaced form: a stored one-token label
|
|
10
|
+
// ("AcmeAI") against the written spaced form ("Acme AI") missed ~93% of real namings
|
|
11
|
+
// (1 of 15 measured).
|
|
12
|
+
// * "AI" never splits internally — the seam is lower/digit → UPPER, so consecutive capitals stay
|
|
13
|
+
// whole and a label like "PopUp" does not become a match for the ordinary phrase "pop up".
|
|
14
|
+
const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
15
|
+
export function buildBrandMatcher(config) {
|
|
16
|
+
const spellings = [config.name, ...(config.aliases ?? [])].filter(Boolean).map(escapeRe);
|
|
17
|
+
const re = new RegExp(`\\b(?:${spellings.join("|")})\\b`, "i");
|
|
18
|
+
const domains = config.domains.map((d) => d.toLowerCase());
|
|
19
|
+
return {
|
|
20
|
+
mentioned: (answer) => re.test(answer),
|
|
21
|
+
sourcedHost: (host) => {
|
|
22
|
+
const h = host.toLowerCase();
|
|
23
|
+
return domains.some((d) => h === d || h.endsWith(`.${d}`));
|
|
24
|
+
},
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
/** Build presence matchers for a competitor set. Tokenizes labels at BOTH kinds of boundary — the
|
|
28
|
+
* separators an engine may drop, and the CamelCase seam an all-together label hides — then rejoins
|
|
29
|
+
* with an optional separator: "AcmeAI" → /\bAcme[.\s-]?AI\b/i, matching "Acme AI", "AcmeAI" and
|
|
30
|
+
* "Acme.AI" alike. The FULL domain is the second alternative, never its root. */
|
|
31
|
+
export function buildCompetitorMatchers(competitors) {
|
|
32
|
+
return competitors.map((c) => {
|
|
33
|
+
const tokens = c.label
|
|
34
|
+
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
|
35
|
+
.split(/[.\s-]+/)
|
|
36
|
+
.filter(Boolean)
|
|
37
|
+
.map(escapeRe);
|
|
38
|
+
const label = tokens.join("[.\\s-]?");
|
|
39
|
+
return { label: c.label, re: new RegExp(`\\b${label}\\b|\\b${escapeRe(c.domain)}\\b`, "i") };
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
/** Competitor labels named in one answer — the atoms of share-of-voice's denominator. A false
|
|
43
|
+
* positive here is indistinguishable from the measured brand losing ground. */
|
|
44
|
+
export function namesIn(answer, matchers) {
|
|
45
|
+
return matchers.filter((c) => c.re.test(answer)).map((c) => c.label);
|
|
46
|
+
}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
/** One atomic observation: a single answer to one prompt on one engine. */
|
|
2
|
+
export type GeoObservation = {
|
|
3
|
+
promptText: string;
|
|
4
|
+
engine: string;
|
|
5
|
+
/** Did this answer actually retrieve? The denominator for `sourced`. */
|
|
6
|
+
grounded: boolean;
|
|
7
|
+
/** Was the measured brand NAMED in the prose? */
|
|
8
|
+
brandMentioned: boolean;
|
|
9
|
+
/** Was one of the brand's own domains among the retrieved sources? null when the answer did not
|
|
10
|
+
* ground. */
|
|
11
|
+
sourced: boolean | null;
|
|
12
|
+
/** Competitor labels named in this one answer. */
|
|
13
|
+
competitors: string[];
|
|
14
|
+
};
|
|
15
|
+
export type GeoWindowSummary = {
|
|
16
|
+
/** Share of voice %: brand presences / (brand + competitor presences), 0-100. NULL when the
|
|
17
|
+
* denominator is empty — nothing was named at all, which is not a zero share. Rounded to 0.1 for
|
|
18
|
+
* storage AFTER the internal share is computed at full precision (Elmo's rule — an intermediate
|
|
19
|
+
* integer percentage rounds small shares away). */
|
|
20
|
+
sovPct: number | null;
|
|
21
|
+
/** Presence rate %: observations naming the brand / total observations, 0-100. NULL when empty. */
|
|
22
|
+
presenceRatePct: number | null;
|
|
23
|
+
/** Sourced rate %: GROUNDED observations where a brand domain was a source / grounded observations.
|
|
24
|
+
* The metric a model cannot fake. NULL when nothing grounded — you cannot be a source of nothing. */
|
|
25
|
+
sourcedRatePct: number | null;
|
|
26
|
+
/** DISTINCT brands named at least once (the measured brand counts only if it appeared). The
|
|
27
|
+
* denominator's WIDTH — a low SoV against 2 rivals and against 20 are different facts. */
|
|
28
|
+
denominatorWidth: number;
|
|
29
|
+
/** Atomic observations pooled = sample size. 0 ⇒ measured nothing. */
|
|
30
|
+
observations: number;
|
|
31
|
+
/** Observations that actually grounded — the honest sourced denominator. */
|
|
32
|
+
groundedObservations: number;
|
|
33
|
+
/** Competitor labels by presence share, most-present first. Counts, not rates, so a reader can see
|
|
34
|
+
* the raw tally. */
|
|
35
|
+
competitors: {
|
|
36
|
+
label: string;
|
|
37
|
+
count: number;
|
|
38
|
+
}[];
|
|
39
|
+
/** 95% confidence intervals (in %) for the three metrics: observation-cluster bootstrap for SoV,
|
|
40
|
+
* Wilson score intervals for the two Bernoulli rates. A number without its interval is a claim
|
|
41
|
+
* without its uncertainty; a thin sample shows a WIDE interval, which is the honest signal. NULL
|
|
42
|
+
* when the metric's denominator is empty. */
|
|
43
|
+
sovCi: Interval | null;
|
|
44
|
+
presenceRateCi: Interval | null;
|
|
45
|
+
sourcedRateCi: Interval | null;
|
|
46
|
+
/** True when ANY measured headline metric is too imprecise: its farthest 95% bound lies more than
|
|
47
|
+
* `MAX_CI_HALF_WIDTH_PP` from the point estimate. A low-confidence number must be shown as a RANGE,
|
|
48
|
+
* never as a headline figure, and a lift claim inside it is unprovable. */
|
|
49
|
+
lowConfidence: boolean;
|
|
50
|
+
};
|
|
51
|
+
/** A confidence interval in percent (0-100), low/high bounds. */
|
|
52
|
+
export type Interval = {
|
|
53
|
+
lo: number;
|
|
54
|
+
hi: number;
|
|
55
|
+
};
|
|
56
|
+
/** The field's replicated noise floor is roughly ±5–7pp. A point estimate is too imprecise to
|
|
57
|
+
* headline when its 95% CI has a half-width above 7pp (14pp full width). */
|
|
58
|
+
export declare const MAX_CI_HALF_WIDTH_PP = 7;
|
|
59
|
+
export declare const MAX_CI_WIDTH_PP: number;
|
|
60
|
+
/** An intervention smaller than 7pp is not practically distinguishable from ordinary run-to-run
|
|
61
|
+
* movement even if a very large sample eventually makes it statistically significant. */
|
|
62
|
+
export declare const MIN_LIFT_PP = 7;
|
|
63
|
+
/** Familywise false-action budget for a frozen prompt-bank version. Repeated looks are handled by the
|
|
64
|
+
* e-process itself; this alpha is divided only across the fixed prompt × engine hypothesis family. */
|
|
65
|
+
export declare const PROMPT_ACTION_FWER = 0.05;
|
|
66
|
+
export type PromptGapEvidence = {
|
|
67
|
+
/** Current mixture e-value. It may fall after a previous crossing; acting on the first crossing is
|
|
68
|
+
* still protected by Ville's inequality. */
|
|
69
|
+
eValue: number;
|
|
70
|
+
/** e-Bonferroni threshold = fixed hypothesis-family size / familywise alpha. */
|
|
71
|
+
threshold: number;
|
|
72
|
+
hypothesisCount: number;
|
|
73
|
+
familywiseAlpha: number;
|
|
74
|
+
proven: boolean;
|
|
75
|
+
};
|
|
76
|
+
/** Anytime-valid evidence that one prompt-engine mention rate is below 50%. The observation stream must
|
|
77
|
+
* be scoped to ONE frozen prompt-bank version and must include only complete scans. Repeated dashboard,
|
|
78
|
+
* MCP and cron looks spend no extra alpha; e-Bonferroni controls the fixed prompt × engine family under
|
|
79
|
+
* arbitrary dependence. Wilson intervals remain a separate display/precision rule. */
|
|
80
|
+
export declare function promptGapEvidence(brandMentions: number, observations: number, hypothesisCount: number, familywiseAlpha?: number): PromptGapEvidence;
|
|
81
|
+
/** Unrounded Wilson bounds in percentage points. Contrast estimators consume these directly so
|
|
82
|
+
* intermediate rounding cannot move a final lift interval across zero. */
|
|
83
|
+
export declare function wilsonCiBounds(successes: number, total: number, z?: number): Interval | null;
|
|
84
|
+
/** 95% Wilson score interval for a binomial proportion `successes/total`, returned in PERCENT and
|
|
85
|
+
* rounded for display/storage. NULL when total=0. Wilson behaves correctly at small n and 0%/100%. */
|
|
86
|
+
export declare function wilsonCi(successes: number, total: number, z?: number): Interval | null;
|
|
87
|
+
/** Full width of an interval in pp; Infinity for a null (unmeasured) interval. */
|
|
88
|
+
export declare const ciWidth: (ci: Interval | null) => number;
|
|
89
|
+
/** The field threshold is about uncertainty around THE estimate, not half of an asymmetric interval's
|
|
90
|
+
* full width. Wilson intervals near 0%/100% are one-sided, so use the farther endpoint. */
|
|
91
|
+
export declare const ciMargin: (estimate: number | null, ci: Interval | null) => number;
|
|
92
|
+
export type PromptGapDecision = {
|
|
93
|
+
status: "unmeasured" | "not_gap" | "research_only" | "proven_gap";
|
|
94
|
+
citedRate: number | null;
|
|
95
|
+
citedRateCi: Interval | null;
|
|
96
|
+
confidenceReady: boolean;
|
|
97
|
+
evidence: PromptGapEvidence;
|
|
98
|
+
reasons: string[];
|
|
99
|
+
};
|
|
100
|
+
/** The one action gate for a prompt on one engine. The Wilson interval answers whether the estimate is
|
|
101
|
+
* precise enough to explain; the e-process answers whether repeated looks plus the fixed family still
|
|
102
|
+
* permit action. Callers must supply the FULL frozen prompt × engine family size, not the number of
|
|
103
|
+
* rows that happened to answer. */
|
|
104
|
+
export declare function evaluatePromptGap(brandMentions: number, observations: number, hypothesisCount: number, familywiseAlpha?: number): PromptGapDecision;
|
|
105
|
+
/** Deterministic cluster bootstrap for share of voice. One answer can name several rivals, so its SoV
|
|
106
|
+
* contributions are not Bernoulli trials and a Wilson interval over raw brand+rival mentions would
|
|
107
|
+
* be falsely narrow. Resampling whole observations preserves that within-answer clustering. A
|
|
108
|
+
* cluster-effective Wilson floor prevents the percentile bootstrap's known zero-width failure when
|
|
109
|
+
* a small sample lands entirely at 0% or 100%. */
|
|
110
|
+
export declare function bootstrapSovCi(observations: readonly GeoObservation[], samples?: number): Interval | null;
|
|
111
|
+
/** Aggregate a window of observations into one summary. Pure. Elmo's `computeReportMetrics` shape:
|
|
112
|
+
* share = brand / (brand + competitors), presence = brand / total, sourced = sourced / grounded, each
|
|
113
|
+
* with an explicit empty-denominator → null (never a divide, never a misleading 0). */
|
|
114
|
+
export declare function computeGeoWindow(observations: readonly GeoObservation[]): GeoWindowSummary;
|
|
115
|
+
/** Per-prompt aggregation for the ACTIVE LOOP: which buyer questions the brand is losing, and to whom,
|
|
116
|
+
* over the window. A prompt is a "gap" when the brand is named in a MINORITY of its observations —
|
|
117
|
+
* pooled, not a single draw. Ranked worst-first: most-uncited, then most-competitors-present. */
|
|
118
|
+
export type GeoPromptWindow = {
|
|
119
|
+
promptText: string;
|
|
120
|
+
observations: number;
|
|
121
|
+
brandMentions: number;
|
|
122
|
+
/** 0-1: brandMentions / observations. Below 0.5 ⇒ we lose this question more often than we win it. */
|
|
123
|
+
citedRate: number;
|
|
124
|
+
/** 95% Wilson interval in 0-1 units for agent-facing ranking and confidence gating. */
|
|
125
|
+
citedRateCi: {
|
|
126
|
+
lo: number;
|
|
127
|
+
hi: number;
|
|
128
|
+
};
|
|
129
|
+
/** True when the rate is wider than the headline precision floor OR its interval crosses the 50%
|
|
130
|
+
* win/loss boundary. A point estimate below 50% is not an actionable loss until its upper bound is
|
|
131
|
+
* also below 50%. */
|
|
132
|
+
lowConfidence: boolean;
|
|
133
|
+
/** Competitor labels named across this prompt's window, by frequency. */
|
|
134
|
+
competitors: {
|
|
135
|
+
label: string;
|
|
136
|
+
count: number;
|
|
137
|
+
}[];
|
|
138
|
+
};
|
|
139
|
+
export declare function computeGeoPromptWindows(observations: readonly GeoObservation[]): GeoPromptWindow[];
|
package/dist/metrics.js
ADDED
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
// geovouch/metrics — the pure math over atomic prompt-run observations. Product-agnostic: see BOUNDARY.md.
|
|
2
|
+
//
|
|
3
|
+
// PORTED FROM Elmo (elmohq/elmo, MIT — packages/lib/src/report-metrics.ts + apps/web/src/lib/
|
|
4
|
+
// visibility-stats.ts), whose design centre is exactly ours: keep the formulas framework-free and
|
|
5
|
+
// DB-free so they run anywhere (a Worker over D1 rows here, a page over Postgres rows there) and are
|
|
6
|
+
// unit-testable without a database. This module is client-safe and imports nothing from lib/server.
|
|
7
|
+
//
|
|
8
|
+
// The reading is a WINDOW aggregate, never a single scan. A grounded answer is non-deterministic
|
|
9
|
+
// (measured: 0.32-0.43 source-set overlap between two runs of one prompt on one engine within 24h), so
|
|
10
|
+
// a single draw per prompt is noise; the field pools repetition over a rolling window. That is the
|
|
11
|
+
// whole reason the storage is one-row-per-observation — see schema/geo.ts.
|
|
12
|
+
/** The field's replicated noise floor is roughly ±5–7pp. A point estimate is too imprecise to
|
|
13
|
+
* headline when its 95% CI has a half-width above 7pp (14pp full width). */
|
|
14
|
+
export const MAX_CI_HALF_WIDTH_PP = 7;
|
|
15
|
+
export const MAX_CI_WIDTH_PP = MAX_CI_HALF_WIDTH_PP * 2;
|
|
16
|
+
/** An intervention smaller than 7pp is not practically distinguishable from ordinary run-to-run
|
|
17
|
+
* movement even if a very large sample eventually makes it statistically significant. */
|
|
18
|
+
export const MIN_LIFT_PP = 7;
|
|
19
|
+
/** Familywise false-action budget for a frozen prompt-bank version. Repeated looks are handled by the
|
|
20
|
+
* e-process itself; this alpha is divided only across the fixed prompt × engine hypothesis family. */
|
|
21
|
+
export const PROMPT_ACTION_FWER = 0.05;
|
|
22
|
+
/** A fixed mixture of plausible loss rates. Each likelihood ratio is a test supermartingale for the
|
|
23
|
+
* composite null p>=50%; averaging them remains anytime-valid without choosing an effect after seeing
|
|
24
|
+
* the answers. This list is policy, so a change requires a prompt-bank-version reset. */
|
|
25
|
+
const PROMPT_GAP_ALTERNATIVES = [0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45];
|
|
26
|
+
/** Anytime-valid evidence that one prompt-engine mention rate is below 50%. The observation stream must
|
|
27
|
+
* be scoped to ONE frozen prompt-bank version and must include only complete scans. Repeated dashboard,
|
|
28
|
+
* MCP and cron looks spend no extra alpha; e-Bonferroni controls the fixed prompt × engine family under
|
|
29
|
+
* arbitrary dependence. Wilson intervals remain a separate display/precision rule. */
|
|
30
|
+
export function promptGapEvidence(brandMentions, observations, hypothesisCount, familywiseAlpha = PROMPT_ACTION_FWER) {
|
|
31
|
+
if (!Number.isSafeInteger(brandMentions) ||
|
|
32
|
+
!Number.isSafeInteger(observations) ||
|
|
33
|
+
brandMentions < 0 ||
|
|
34
|
+
observations < 0 ||
|
|
35
|
+
brandMentions > observations)
|
|
36
|
+
throw new RangeError("brandMentions and observations must be valid integer counts");
|
|
37
|
+
if (!Number.isSafeInteger(hypothesisCount) || hypothesisCount <= 0) {
|
|
38
|
+
throw new RangeError("hypothesisCount must be a positive safe integer");
|
|
39
|
+
}
|
|
40
|
+
if (!Number.isFinite(familywiseAlpha) || familywiseAlpha <= 0 || familywiseAlpha >= 1) {
|
|
41
|
+
throw new RangeError("familywiseAlpha must be between 0 and 1");
|
|
42
|
+
}
|
|
43
|
+
const failures = observations - brandMentions;
|
|
44
|
+
const logTerms = PROMPT_GAP_ALTERNATIVES.map((alternative) => brandMentions * Math.log(alternative / 0.5) +
|
|
45
|
+
failures * Math.log((1 - alternative) / 0.5));
|
|
46
|
+
const maxLog = Math.max(...logTerms);
|
|
47
|
+
const logEValue = maxLog +
|
|
48
|
+
Math.log(logTerms.reduce((total, value) => total + Math.exp(value - maxLog), 0) / PROMPT_GAP_ALTERNATIVES.length);
|
|
49
|
+
const threshold = hypothesisCount / familywiseAlpha;
|
|
50
|
+
return {
|
|
51
|
+
eValue: Math.exp(Math.min(logEValue, Math.log(Number.MAX_VALUE))),
|
|
52
|
+
threshold,
|
|
53
|
+
hypothesisCount,
|
|
54
|
+
familywiseAlpha,
|
|
55
|
+
proven: logEValue >= Math.log(threshold),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
const pct = (num, den) => (den <= 0 ? null : Math.round((num / den) * 1000) / 10);
|
|
59
|
+
/** Unrounded Wilson bounds in percentage points. Contrast estimators consume these directly so
|
|
60
|
+
* intermediate rounding cannot move a final lift interval across zero. */
|
|
61
|
+
export function wilsonCiBounds(successes, total, z = 1.96) {
|
|
62
|
+
if (!Number.isFinite(successes) || !Number.isFinite(total) || !Number.isFinite(z) || total <= 0 || successes < 0 || successes > total || z <= 0)
|
|
63
|
+
return null;
|
|
64
|
+
const p = successes / total;
|
|
65
|
+
const z2 = z * z;
|
|
66
|
+
const denom = 1 + z2 / total;
|
|
67
|
+
const center = (p + z2 / (2 * total)) / denom;
|
|
68
|
+
const half = (z / denom) * Math.sqrt((p * (1 - p)) / total + z2 / (4 * total * total));
|
|
69
|
+
const clamp = (x) => Math.max(0, Math.min(1, x)) * 100;
|
|
70
|
+
return { lo: clamp(center - half), hi: clamp(center + half) };
|
|
71
|
+
}
|
|
72
|
+
/** 95% Wilson score interval for a binomial proportion `successes/total`, returned in PERCENT and
|
|
73
|
+
* rounded for display/storage. NULL when total=0. Wilson behaves correctly at small n and 0%/100%. */
|
|
74
|
+
export function wilsonCi(successes, total, z = 1.96) {
|
|
75
|
+
const ci = wilsonCiBounds(successes, total, z);
|
|
76
|
+
return ci ? { lo: Math.round(ci.lo * 10) / 10, hi: Math.round(ci.hi * 10) / 10 } : null;
|
|
77
|
+
}
|
|
78
|
+
/** Full width of an interval in pp; Infinity for a null (unmeasured) interval. */
|
|
79
|
+
export const ciWidth = (ci) => (ci ? ci.hi - ci.lo : Infinity);
|
|
80
|
+
/** The field threshold is about uncertainty around THE estimate, not half of an asymmetric interval's
|
|
81
|
+
* full width. Wilson intervals near 0%/100% are one-sided, so use the farther endpoint. */
|
|
82
|
+
export const ciMargin = (estimate, ci) => estimate === null || !ci ? Infinity : Math.max(estimate - ci.lo, ci.hi - estimate);
|
|
83
|
+
/** The one action gate for a prompt on one engine. The Wilson interval answers whether the estimate is
|
|
84
|
+
* precise enough to explain; the e-process answers whether repeated looks plus the fixed family still
|
|
85
|
+
* permit action. Callers must supply the FULL frozen prompt × engine family size, not the number of
|
|
86
|
+
* rows that happened to answer. */
|
|
87
|
+
export function evaluatePromptGap(brandMentions, observations, hypothesisCount, familywiseAlpha = PROMPT_ACTION_FWER) {
|
|
88
|
+
const evidence = promptGapEvidence(brandMentions, observations, hypothesisCount, familywiseAlpha);
|
|
89
|
+
const estimatePct = observations > 0 ? (brandMentions / observations) * 100 : null;
|
|
90
|
+
const bounds = wilsonCiBounds(brandMentions, observations);
|
|
91
|
+
const citedRateCi = bounds
|
|
92
|
+
? {
|
|
93
|
+
lo: Math.round(bounds.lo * 10) / 10,
|
|
94
|
+
hi: Math.round(bounds.hi * 10) / 10,
|
|
95
|
+
}
|
|
96
|
+
: null;
|
|
97
|
+
const citedRate = estimatePct === null ? null : estimatePct / 100;
|
|
98
|
+
const confidenceReady = estimatePct !== null &&
|
|
99
|
+
bounds !== null &&
|
|
100
|
+
bounds.hi < 50 &&
|
|
101
|
+
ciMargin(estimatePct, bounds) <= MAX_CI_HALF_WIDTH_PP;
|
|
102
|
+
if (estimatePct === null) {
|
|
103
|
+
return {
|
|
104
|
+
status: "unmeasured",
|
|
105
|
+
citedRate,
|
|
106
|
+
citedRateCi,
|
|
107
|
+
confidenceReady,
|
|
108
|
+
evidence,
|
|
109
|
+
reasons: ["no observations in the current prompt-bank version"],
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
if (estimatePct >= 50) {
|
|
113
|
+
return {
|
|
114
|
+
status: "not_gap",
|
|
115
|
+
citedRate,
|
|
116
|
+
citedRateCi,
|
|
117
|
+
confidenceReady: false,
|
|
118
|
+
evidence,
|
|
119
|
+
reasons: ["the brand is named in at least half of observations"],
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
const reasons = [];
|
|
123
|
+
if (!bounds || bounds.hi >= 50) {
|
|
124
|
+
reasons.push("the 95% Wilson interval does not lie wholly below 50%");
|
|
125
|
+
}
|
|
126
|
+
if (!bounds || ciMargin(estimatePct, bounds) > MAX_CI_HALF_WIDTH_PP) {
|
|
127
|
+
reasons.push(`the 95% interval extends more than ±${MAX_CI_HALF_WIDTH_PP}pp from the estimate`);
|
|
128
|
+
}
|
|
129
|
+
if (!evidence.proven) {
|
|
130
|
+
reasons.push(`the anytime-valid familywise e-value ${evidence.eValue.toPrecision(3)} is below ${evidence.threshold.toFixed(0)}`);
|
|
131
|
+
}
|
|
132
|
+
return {
|
|
133
|
+
status: confidenceReady && evidence.proven ? "proven_gap" : "research_only",
|
|
134
|
+
citedRate,
|
|
135
|
+
citedRateCi,
|
|
136
|
+
confidenceReady,
|
|
137
|
+
evidence,
|
|
138
|
+
reasons,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
/** Deterministic cluster bootstrap for share of voice. One answer can name several rivals, so its SoV
|
|
142
|
+
* contributions are not Bernoulli trials and a Wilson interval over raw brand+rival mentions would
|
|
143
|
+
* be falsely narrow. Resampling whole observations preserves that within-answer clustering. A
|
|
144
|
+
* cluster-effective Wilson floor prevents the percentile bootstrap's known zero-width failure when
|
|
145
|
+
* a small sample lands entirely at 0% or 100%. */
|
|
146
|
+
export function bootstrapSovCi(observations, samples = 10_000) {
|
|
147
|
+
const contributions = observations
|
|
148
|
+
.map((o) => ({ brand: o.brandMentioned ? 1 : 0, rivals: o.competitors.length }))
|
|
149
|
+
.sort((a, b) => a.brand - b.brand || a.rivals - b.rivals);
|
|
150
|
+
const brand = contributions.reduce((total, x) => total + x.brand, 0);
|
|
151
|
+
const mentionWeights = contributions.map((x) => x.brand + x.rivals);
|
|
152
|
+
const mentions = mentionWeights.reduce((total, weight) => total + weight, 0);
|
|
153
|
+
if (mentions === 0 || samples <= 0)
|
|
154
|
+
return null;
|
|
155
|
+
let state = 0x9e3779b9;
|
|
156
|
+
const shares = [];
|
|
157
|
+
for (let sample = 0; sample < samples; sample++) {
|
|
158
|
+
let sampleBrand = 0;
|
|
159
|
+
let sampleRivals = 0;
|
|
160
|
+
for (let draw = 0; draw < contributions.length; draw++) {
|
|
161
|
+
state ^= state << 13;
|
|
162
|
+
state ^= state >>> 17;
|
|
163
|
+
state ^= state << 5;
|
|
164
|
+
const picked = contributions[(state >>> 0) % contributions.length];
|
|
165
|
+
sampleBrand += picked.brand;
|
|
166
|
+
sampleRivals += picked.rivals;
|
|
167
|
+
}
|
|
168
|
+
if (sampleBrand + sampleRivals > 0)
|
|
169
|
+
shares.push((sampleBrand / (sampleBrand + sampleRivals)) * 100);
|
|
170
|
+
}
|
|
171
|
+
if (!shares.length)
|
|
172
|
+
return null;
|
|
173
|
+
shares.sort((a, b) => a - b);
|
|
174
|
+
const quantile = (q) => shares[Math.min(shares.length - 1, Math.max(0, Math.floor(q * shares.length)))];
|
|
175
|
+
const bootstrap = { lo: quantile(0.025), hi: quantile(0.975) };
|
|
176
|
+
// Kish effective sample size treats each answer as one cluster and discounts answers that contribute
|
|
177
|
+
// many rival mentions. Fractional Wilson successes are the SoV estimate at that effective n.
|
|
178
|
+
const squaredWeights = mentionWeights.reduce((total, weight) => total + weight * weight, 0);
|
|
179
|
+
const effectiveN = (mentions * mentions) / squaredWeights;
|
|
180
|
+
const floor = wilsonCi((brand / mentions) * effectiveN, effectiveN);
|
|
181
|
+
return {
|
|
182
|
+
lo: Math.round(Math.min(bootstrap.lo, floor.lo) * 10) / 10,
|
|
183
|
+
hi: Math.round(Math.max(bootstrap.hi, floor.hi) * 10) / 10,
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
/** Aggregate a window of observations into one summary. Pure. Elmo's `computeReportMetrics` shape:
|
|
187
|
+
* share = brand / (brand + competitors), presence = brand / total, sourced = sourced / grounded, each
|
|
188
|
+
* with an explicit empty-denominator → null (never a divide, never a misleading 0). */
|
|
189
|
+
export function computeGeoWindow(observations) {
|
|
190
|
+
let brandPresences = 0;
|
|
191
|
+
let competitorPresences = 0;
|
|
192
|
+
let groundedObservations = 0;
|
|
193
|
+
let sourcedObservations = 0;
|
|
194
|
+
const competitorsSeen = new Set();
|
|
195
|
+
const competitorCounts = new Map();
|
|
196
|
+
for (const o of observations) {
|
|
197
|
+
if (o.brandMentioned)
|
|
198
|
+
brandPresences += 1;
|
|
199
|
+
for (const label of o.competitors) {
|
|
200
|
+
competitorPresences += 1;
|
|
201
|
+
competitorsSeen.add(label.toLowerCase());
|
|
202
|
+
competitorCounts.set(label, (competitorCounts.get(label) ?? 0) + 1);
|
|
203
|
+
}
|
|
204
|
+
if (o.grounded) {
|
|
205
|
+
groundedObservations += 1;
|
|
206
|
+
if (o.sourced === true)
|
|
207
|
+
sourcedObservations += 1;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
const total = observations.length;
|
|
211
|
+
const sovPct = pct(brandPresences, brandPresences + competitorPresences);
|
|
212
|
+
const presenceRatePct = pct(brandPresences, total);
|
|
213
|
+
const sourcedRatePct = pct(sourcedObservations, groundedObservations);
|
|
214
|
+
const sovCi = bootstrapSovCi(observations);
|
|
215
|
+
const presenceRateCi = wilsonCi(brandPresences, total);
|
|
216
|
+
const sourcedRateCi = wilsonCi(sourcedObservations, groundedObservations);
|
|
217
|
+
return {
|
|
218
|
+
sovPct,
|
|
219
|
+
presenceRatePct,
|
|
220
|
+
sourcedRatePct,
|
|
221
|
+
denominatorWidth: competitorsSeen.size + (brandPresences > 0 ? 1 : 0),
|
|
222
|
+
observations: total,
|
|
223
|
+
groundedObservations,
|
|
224
|
+
competitors: [...competitorCounts.entries()]
|
|
225
|
+
.map(([label, count]) => ({ label, count }))
|
|
226
|
+
.sort((a, b) => b.count - a.count || a.label.localeCompare(b.label)),
|
|
227
|
+
sovCi,
|
|
228
|
+
presenceRateCi,
|
|
229
|
+
sourcedRateCi,
|
|
230
|
+
// A window with NO observations makes every metric null, so every `x !== null` test is false and
|
|
231
|
+
// the OR was FALSE — an engine that measured nothing reported as high-confidence, which the header
|
|
232
|
+
// then persisted and the MCP scoreboard rendered as "every metric is within ±7pp". Nothing measured
|
|
233
|
+
// is the least confident state there is.
|
|
234
|
+
lowConfidence: total === 0 ||
|
|
235
|
+
(sovPct !== null && ciMargin(sovPct, sovCi) > MAX_CI_HALF_WIDTH_PP) ||
|
|
236
|
+
(presenceRatePct !== null && ciMargin(presenceRatePct, presenceRateCi) > MAX_CI_HALF_WIDTH_PP) ||
|
|
237
|
+
(sourcedRatePct !== null && ciMargin(sourcedRatePct, sourcedRateCi) > MAX_CI_HALF_WIDTH_PP),
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
export function computeGeoPromptWindows(observations) {
|
|
241
|
+
const byPrompt = new Map();
|
|
242
|
+
for (const o of observations) {
|
|
243
|
+
const arr = byPrompt.get(o.promptText);
|
|
244
|
+
if (arr)
|
|
245
|
+
arr.push(o);
|
|
246
|
+
else
|
|
247
|
+
byPrompt.set(o.promptText, [o]);
|
|
248
|
+
}
|
|
249
|
+
const out = [];
|
|
250
|
+
for (const [promptText, obs] of byPrompt) {
|
|
251
|
+
const brandMentions = obs.filter((o) => o.brandMentioned).length;
|
|
252
|
+
const counts = new Map();
|
|
253
|
+
for (const o of obs)
|
|
254
|
+
for (const c of o.competitors)
|
|
255
|
+
counts.set(c, (counts.get(c) ?? 0) + 1);
|
|
256
|
+
const citedRateCiPct = wilsonCi(brandMentions, obs.length) ?? { lo: 0, hi: 100 };
|
|
257
|
+
const citedRate = obs.length > 0 ? brandMentions / obs.length : 0;
|
|
258
|
+
const crossesDecisionBoundary = citedRate < 0.5 ? citedRateCiPct.hi >= 50 : citedRateCiPct.lo < 50;
|
|
259
|
+
out.push({
|
|
260
|
+
promptText,
|
|
261
|
+
observations: obs.length,
|
|
262
|
+
brandMentions,
|
|
263
|
+
citedRate,
|
|
264
|
+
citedRateCi: {
|
|
265
|
+
lo: Math.round(citedRateCiPct.lo * 10) / 1000,
|
|
266
|
+
hi: Math.round(citedRateCiPct.hi * 10) / 1000,
|
|
267
|
+
},
|
|
268
|
+
lowConfidence: ciMargin(citedRate * 100, citedRateCiPct) > MAX_CI_HALF_WIDTH_PP || crossesDecisionBoundary,
|
|
269
|
+
competitors: [...counts.entries()].map(([label, count]) => ({ label, count })).sort((a, b) => b.count - a.count),
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
return out.sort((a, b) => a.citedRate - b.citedRate || b.competitors.length - a.competitors.length);
|
|
273
|
+
}
|
package/dist/runner.d.ts
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { type BrandConfig } from "./matchers.ts";
|
|
2
|
+
import type { GeoObservation } from "./metrics.ts";
|
|
3
|
+
/** One measured row plus the non-metric facts a consumer usually stores beside it. */
|
|
4
|
+
export type GeoScanRow = GeoObservation & {
|
|
5
|
+
citedHosts: string[];
|
|
6
|
+
searchQueries: number;
|
|
7
|
+
};
|
|
8
|
+
export type GeoScanConfig = {
|
|
9
|
+
brand: BrandConfig;
|
|
10
|
+
competitors: readonly {
|
|
11
|
+
domain: string;
|
|
12
|
+
label: string;
|
|
13
|
+
}[];
|
|
14
|
+
prompts: readonly string[];
|
|
15
|
+
/** The row's engine label, e.g. "gemini-grounded". The aggregator groups by it and never averages
|
|
16
|
+
* across engines — they diverge wildly. */
|
|
17
|
+
engine: string;
|
|
18
|
+
/** Whether `ask` runs with retrieval. Controls both the system nudge and whether grounding metadata
|
|
19
|
+
* is read. An ungrounded scan's `sourced` is always null (unmeasured, never false). */
|
|
20
|
+
grounded: boolean;
|
|
21
|
+
/** Answers per prompt per scan. Default 1 — a rolling window across scans pools better than a burst
|
|
22
|
+
* on one day (portfolio-level, measured by the field); raise only alongside a cadence change. */
|
|
23
|
+
repetitions?: number;
|
|
24
|
+
/** Override the system prompt. The default asks for concrete app/product recommendations and, when
|
|
25
|
+
* grounded, to search first. */
|
|
26
|
+
system?: {
|
|
27
|
+
grounded?: string;
|
|
28
|
+
ungrounded?: string;
|
|
29
|
+
};
|
|
30
|
+
};
|
|
31
|
+
export type GeoScanPorts = {
|
|
32
|
+
/** One LLM call. Throwing loses this observation only. */
|
|
33
|
+
ask(prompt: string, system: string): Promise<{
|
|
34
|
+
text?: string | null;
|
|
35
|
+
providerMetadata?: unknown;
|
|
36
|
+
}>;
|
|
37
|
+
/** Persist one row as it is measured. Catch your own storage errors — a throw here also costs only
|
|
38
|
+
* this observation. */
|
|
39
|
+
onObservation?(row: GeoScanRow): void | Promise<void>;
|
|
40
|
+
/** Polled before every call; true stops the scan with a stoppedReason. */
|
|
41
|
+
budget?: {
|
|
42
|
+
overBudget(): boolean;
|
|
43
|
+
};
|
|
44
|
+
};
|
|
45
|
+
export type GeoScanResult = {
|
|
46
|
+
observations: GeoScanRow[];
|
|
47
|
+
/** Raw answer texts, for a consumer's own mining pass. Never persisted by the kit. */
|
|
48
|
+
answers: {
|
|
49
|
+
prompt: string;
|
|
50
|
+
answer: string;
|
|
51
|
+
}[];
|
|
52
|
+
/** Retrieved hosts per prompt (accumulated across repetitions) — venue analysis reads this. */
|
|
53
|
+
hostsByPrompt: Map<string, string[]>;
|
|
54
|
+
/** Billable search queries across the scan — the only honest bound on a grounded bill. */
|
|
55
|
+
searchQueries: number;
|
|
56
|
+
stoppedReason: string | null;
|
|
57
|
+
};
|
|
58
|
+
export declare function runGeoScan(config: GeoScanConfig, ports: GeoScanPorts): Promise<GeoScanResult>;
|
package/dist/runner.js
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// geovouch/runner — the generic scan loop, with every side effect injected as a port. The kit stays
|
|
2
|
+
// IO-free by construction: it never imports an LLM client, a database or a clock — the consumer hands
|
|
3
|
+
// in `ask` (one LLM call) and optionally `onObservation` (persist one row) and `budget` (spend guard),
|
|
4
|
+
// and the runner owns only the loop's SHAPE: repetition, fail-soft per observation, grounded-answer
|
|
5
|
+
// parsing, brand/competitor detection.
|
|
6
|
+
//
|
|
7
|
+
// This shape is lifted from the first consumer's production loop, where it ran daily against grounded
|
|
8
|
+
// Gemini. What each port is for, learned there:
|
|
9
|
+
// * `ask` returns the raw text + providerMetadata; a throw inside it loses ONE observation, never
|
|
10
|
+
// the scan (no fake data is ever recorded for a failed call).
|
|
11
|
+
// * `budget.overBudget()` is polled before every call, because a grounded prompt's search fan-out is
|
|
12
|
+
// the model's choice, not the caller's — the bank size does not bound the bill; only counting the
|
|
13
|
+
// answers does (see gemini.ts).
|
|
14
|
+
// * `onObservation` receives each row as it is measured, so storage is append-per-row and a failed
|
|
15
|
+
// insert (the port's own catch) costs one row, never the scan.
|
|
16
|
+
import { buildBrandMatcher, buildCompetitorMatchers, namesIn } from "./matchers.js";
|
|
17
|
+
import { groundedHosts, groundedQueryCount } from "./gemini.js";
|
|
18
|
+
const DEFAULT_GROUNDED_SYSTEM = "You are a helpful assistant recommending apps. SEARCH THE WEB FIRST, then answer naturally and concretely, naming specific apps you'd recommend based on what you found.";
|
|
19
|
+
const DEFAULT_UNGROUNDED_SYSTEM = "You are a helpful assistant recommending apps. Answer naturally and concretely, naming specific apps you'd recommend.";
|
|
20
|
+
export async function runGeoScan(config, ports) {
|
|
21
|
+
const brand = buildBrandMatcher(config.brand);
|
|
22
|
+
const matchers = buildCompetitorMatchers(config.competitors);
|
|
23
|
+
const repetitions = config.repetitions ?? 1;
|
|
24
|
+
const system = config.grounded
|
|
25
|
+
? (config.system?.grounded ?? DEFAULT_GROUNDED_SYSTEM)
|
|
26
|
+
: (config.system?.ungrounded ?? DEFAULT_UNGROUNDED_SYSTEM);
|
|
27
|
+
const observations = [];
|
|
28
|
+
const answers = [];
|
|
29
|
+
const hostsByPrompt = new Map();
|
|
30
|
+
let searchQueries = 0;
|
|
31
|
+
let stoppedReason = null;
|
|
32
|
+
for (const prompt of config.prompts) {
|
|
33
|
+
for (let rep = 0; rep < repetitions; rep++) {
|
|
34
|
+
if (ports.budget?.overBudget()) {
|
|
35
|
+
stoppedReason = `budget reached after ${searchQueries} search queries`;
|
|
36
|
+
break;
|
|
37
|
+
}
|
|
38
|
+
let answer = "";
|
|
39
|
+
let hosts = [];
|
|
40
|
+
let queries = 0;
|
|
41
|
+
try {
|
|
42
|
+
const { text, providerMetadata } = await ports.ask(prompt, system);
|
|
43
|
+
answer = text ?? "";
|
|
44
|
+
hosts = config.grounded ? groundedHosts(providerMetadata) : [];
|
|
45
|
+
queries = config.grounded ? groundedQueryCount(providerMetadata) : 0;
|
|
46
|
+
searchQueries += queries;
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
continue; // this observation just doesn't contribute (no fake data)
|
|
50
|
+
}
|
|
51
|
+
// Grounding is per-ANSWER: a memory answer inside a grounded scan retrieved nothing, so its
|
|
52
|
+
// sourced is null (unmeasured), never false.
|
|
53
|
+
const grounded = queries > 0 || hosts.length > 0;
|
|
54
|
+
const row = {
|
|
55
|
+
promptText: prompt,
|
|
56
|
+
engine: config.engine,
|
|
57
|
+
grounded,
|
|
58
|
+
brandMentioned: brand.mentioned(answer),
|
|
59
|
+
sourced: grounded ? hosts.some((h) => brand.sourcedHost(h)) : null,
|
|
60
|
+
competitors: namesIn(answer, matchers),
|
|
61
|
+
citedHosts: hosts,
|
|
62
|
+
searchQueries: queries,
|
|
63
|
+
};
|
|
64
|
+
observations.push(row);
|
|
65
|
+
if (answer)
|
|
66
|
+
answers.push({ prompt, answer });
|
|
67
|
+
if (hosts.length)
|
|
68
|
+
hostsByPrompt.set(prompt, [...(hostsByPrompt.get(prompt) ?? []), ...hosts]);
|
|
69
|
+
try {
|
|
70
|
+
await ports.onObservation?.(row);
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
/* a failed persist costs one row, never the scan */
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
if (stoppedReason)
|
|
77
|
+
break;
|
|
78
|
+
}
|
|
79
|
+
return { observations, answers, hostsByPrompt, searchQueries, stoppedReason };
|
|
80
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "geovouch",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "GeoVouch measurement core — GEO (Generative Engine Optimization) share-of-voice over atomic prompt observations: window metrics, brand/competitor matchers, grounding-metadata parsing, and a port-injected scan runner. Pure TypeScript, zero dependencies, zero IO.",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"engines": {
|
|
8
|
+
"node": ">=20"
|
|
9
|
+
},
|
|
10
|
+
"files": [
|
|
11
|
+
"dist"
|
|
12
|
+
],
|
|
13
|
+
"exports": {
|
|
14
|
+
"./metrics": {
|
|
15
|
+
"types": "./dist/metrics.d.ts",
|
|
16
|
+
"default": "./dist/metrics.js"
|
|
17
|
+
},
|
|
18
|
+
"./matchers": {
|
|
19
|
+
"types": "./dist/matchers.d.ts",
|
|
20
|
+
"default": "./dist/matchers.js"
|
|
21
|
+
},
|
|
22
|
+
"./gemini": {
|
|
23
|
+
"types": "./dist/gemini.d.ts",
|
|
24
|
+
"default": "./dist/gemini.js"
|
|
25
|
+
},
|
|
26
|
+
"./runner": {
|
|
27
|
+
"types": "./dist/runner.d.ts",
|
|
28
|
+
"default": "./dist/runner.js"
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
"scripts": {
|
|
32
|
+
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
33
|
+
"build": "tsc -p tsconfig.build.json",
|
|
34
|
+
"test": "node --test --experimental-transform-types test/*.test.ts",
|
|
35
|
+
"boundary": "node ../../scripts/check-boundary.mjs"
|
|
36
|
+
}
|
|
37
|
+
}
|