fullstackgtm 0.48.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +91 -0
- package/CONTRIBUTING.md +23 -4
- package/DATA-FLOWS.md +8 -2
- package/INSTALL_FOR_AGENTS.md +13 -4
- package/README.md +53 -1
- package/SECURITY.md +27 -3
- package/dist/acquireCheckpoint.d.ts +33 -0
- package/dist/acquireCheckpoint.js +149 -0
- package/dist/acquireLinkedIn.d.ts +2 -0
- package/dist/acquireLinkedIn.js +1 -1
- package/dist/cli/audit.js +10 -7
- package/dist/cli/auth.js +8 -4
- package/dist/cli/enrich.js +283 -79
- package/dist/cli/fix.js +20 -7
- package/dist/cli/help.js +23 -15
- package/dist/cli/market.js +180 -2
- package/dist/cli/plans.js +154 -9
- package/dist/cli/shared.d.ts +3 -1
- package/dist/cli/shared.js +2 -2
- package/dist/cli/ui.d.ts +10 -5
- package/dist/cli/ui.js +18 -5
- package/dist/cli.js +1 -1
- package/dist/config.d.ts +13 -1
- package/dist/config.js +23 -2
- package/dist/connectors/linkedin.d.ts +2 -0
- package/dist/connectors/linkedin.js +5 -0
- package/dist/connectors/prospectSources.d.ts +23 -0
- package/dist/connectors/prospectSources.js +25 -14
- package/dist/connectors/salesforce.js +12 -5
- package/dist/connectors/salesforceAuth.d.ts +9 -0
- package/dist/connectors/salesforceAuth.js +47 -5
- package/dist/connectors/theirstack.d.ts +0 -19
- package/dist/connectors/theirstack.js +3 -10
- package/dist/credentials.js +36 -26
- package/dist/enrich.d.ts +44 -1
- package/dist/hostedAcquireCheckpoint.d.ts +43 -0
- package/dist/hostedAcquireCheckpoint.js +129 -0
- package/dist/hostedPatchPlan.d.ts +87 -0
- package/dist/hostedPatchPlan.js +270 -0
- package/dist/icp.js +13 -2
- package/dist/index.d.ts +7 -3
- package/dist/index.js +6 -2
- package/dist/market.d.ts +1 -1
- package/dist/market.js +7 -127
- package/dist/marketClassify.d.ts +10 -0
- package/dist/marketClassify.js +52 -1
- package/dist/marketSourcing.js +7 -45
- package/dist/mcp.js +67 -115
- package/dist/planStore.d.ts +42 -1
- package/dist/planStore.js +268 -49
- package/dist/progress.d.ts +2 -2
- package/dist/progress.js +2 -2
- package/dist/providerError.d.ts +21 -0
- package/dist/providerError.js +30 -0
- package/dist/publicHttp.d.ts +28 -0
- package/dist/publicHttp.js +143 -0
- package/dist/secureFile.d.ts +15 -0
- package/dist/secureFile.js +164 -0
- package/dist/types.d.ts +1 -1
- package/docs/api.md +53 -2
- package/docs/architecture.md +22 -2
- package/llms.txt +7 -0
- package/package.json +5 -3
- package/skills/fullstackgtm/SKILL.md +8 -2
- package/src/acquireCheckpoint.ts +186 -0
- package/src/acquireLinkedIn.ts +3 -1
- package/src/cli/audit.ts +10 -7
- package/src/cli/auth.ts +8 -4
- package/src/cli/enrich.ts +322 -78
- package/src/cli/fix.ts +28 -7
- package/src/cli/help.ts +24 -15
- package/src/cli/market.ts +181 -2
- package/src/cli/plans.ts +159 -9
- package/src/cli/shared.ts +2 -1
- package/src/cli/ui.ts +19 -9
- package/src/cli.ts +1 -1
- package/src/config.ts +39 -1
- package/src/connectors/linkedin.ts +6 -0
- package/src/connectors/prospectSources.ts +50 -15
- package/src/connectors/salesforce.ts +12 -5
- package/src/connectors/salesforceAuth.ts +46 -6
- package/src/connectors/theirstack.ts +3 -10
- package/src/credentials.ts +47 -28
- package/src/enrich.ts +47 -1
- package/src/hostedAcquireCheckpoint.ts +156 -0
- package/src/hostedPatchPlan.ts +286 -0
- package/src/icp.ts +14 -2
- package/src/index.ts +24 -0
- package/src/market.ts +7 -111
- package/src/marketClassify.ts +61 -1
- package/src/marketSourcing.ts +7 -43
- package/src/mcp.ts +93 -136
- package/src/planStore.ts +322 -57
- package/src/progress.ts +2 -2
- package/src/providerError.ts +37 -0
- package/src/publicHttp.ts +147 -0
- package/src/secureFile.ts +169 -0
- package/src/types.ts +1 -0
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
/** Broker-authenticated mirror for locally governed patch plans. */
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { getCredential } from "./credentials.js";
|
|
4
|
+
const ENDPOINT = "/api/cli/patch-plan";
|
|
5
|
+
const TIMEOUT_MS = 5000;
|
|
6
|
+
function broker() {
|
|
7
|
+
const value = getCredential("broker");
|
|
8
|
+
if (!value?.baseUrl || !value.accessToken)
|
|
9
|
+
return null;
|
|
10
|
+
try {
|
|
11
|
+
const url = new URL(value.baseUrl);
|
|
12
|
+
const local = ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname);
|
|
13
|
+
if (url.protocol !== "https:" && !(url.protocol === "http:" && local))
|
|
14
|
+
return null;
|
|
15
|
+
return { baseUrl: url.href.replace(/\/+$/, ""), token: value.accessToken };
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return null;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
/** Review document intentionally omits evidence/findings and all mutable store security state. */
|
|
22
|
+
export function hostedReviewDocument(plan) {
|
|
23
|
+
return {
|
|
24
|
+
id: plan.id,
|
|
25
|
+
title: plan.title,
|
|
26
|
+
createdAt: plan.createdAt,
|
|
27
|
+
status: "needs_approval",
|
|
28
|
+
dryRun: true,
|
|
29
|
+
summary: plan.summary,
|
|
30
|
+
operations: plan.operations,
|
|
31
|
+
...(plan.filter ? { filter: plan.filter } : {}),
|
|
32
|
+
...(plan.guards ? { guards: plan.guards } : {}),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
export function hostedPlanDigest(plan) {
|
|
36
|
+
return createHash("sha256").update(JSON.stringify(hostedReviewDocument(plan))).digest("hex");
|
|
37
|
+
}
|
|
38
|
+
function mapOperation(op) {
|
|
39
|
+
return {
|
|
40
|
+
packageOpId: op.id,
|
|
41
|
+
objectType: op.objectType,
|
|
42
|
+
...(op.objectId ? { objectId: op.objectId } : {}),
|
|
43
|
+
fieldOrAction: op.field ?? op.operation,
|
|
44
|
+
beforeValue: op.beforeValue ?? null,
|
|
45
|
+
afterValue: op.afterValue ?? null,
|
|
46
|
+
...(op.reason ? { reason: op.reason } : {}),
|
|
47
|
+
...(op.sourceRuleOrPolicy ? { sourceRuleOrPolicy: op.sourceRuleOrPolicy } : {}),
|
|
48
|
+
riskLevel: op.riskLevel,
|
|
49
|
+
approvalRequired: op.approvalRequired,
|
|
50
|
+
...(op.rollback ? { rollback: op.rollback } : {}),
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
function parseState(value, baseUrl) {
|
|
54
|
+
if (!value || typeof value !== "object")
|
|
55
|
+
return null;
|
|
56
|
+
const row = value;
|
|
57
|
+
const id = typeof row.patchPlanId === "string" ? row.patchPlanId : null;
|
|
58
|
+
if (!id)
|
|
59
|
+
return null;
|
|
60
|
+
const approvedOperationIds = Array.isArray(row.approvedOperationIds) ? row.approvedOperationIds.filter((id) => typeof id === "string") : [];
|
|
61
|
+
const runRecord = parseRunRecord(row.runRecord, row.operationReceipts, row.externalPlanId, row.updatedAt);
|
|
62
|
+
return {
|
|
63
|
+
patchPlanId: id,
|
|
64
|
+
externalPlanId: typeof row.externalPlanId === "string" ? row.externalPlanId : "",
|
|
65
|
+
documentSha256: typeof row.documentSha256 === "string" ? row.documentSha256 : "",
|
|
66
|
+
status: typeof row.status === "string" ? row.status : "pending_approval",
|
|
67
|
+
approvedOperationIds,
|
|
68
|
+
valueOverrides: row.valueOverrides && typeof row.valueOverrides === "object" && !Array.isArray(row.valueOverrides) ? row.valueOverrides : {},
|
|
69
|
+
updatedAt: typeof row.updatedAt === "number" ? row.updatedAt : Date.now(),
|
|
70
|
+
...(runRecord ? { runRecord } : {}),
|
|
71
|
+
url: `${baseUrl}/dashboard/patch-plans?plan=${encodeURIComponent(id)}`,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
function parseRunRecord(value, receiptsValue, externalPlanId, updatedAt) {
|
|
75
|
+
const row = value && typeof value === "object" && !Array.isArray(value) ? value : undefined;
|
|
76
|
+
const planId = typeof row?.planId === "string" ? row.planId : typeof externalPlanId === "string" ? externalPlanId : undefined;
|
|
77
|
+
if (!planId)
|
|
78
|
+
return undefined;
|
|
79
|
+
const rawResults = Array.isArray(row?.results) ? row.results : Array.isArray(receiptsValue) ? receiptsValue : [];
|
|
80
|
+
const results = rawResults.flatMap((entry) => {
|
|
81
|
+
if (!entry || typeof entry !== "object")
|
|
82
|
+
return [];
|
|
83
|
+
const receipt = entry;
|
|
84
|
+
const operationId = typeof receipt.operationId === "string" ? receipt.operationId : typeof receipt.packageOpId === "string" ? receipt.packageOpId : undefined;
|
|
85
|
+
const rawStatus = receipt.status;
|
|
86
|
+
const status = rawStatus === "applied" || rawStatus === "failed" || rawStatus === "skipped" || rawStatus === "conflict" ? rawStatus : undefined;
|
|
87
|
+
if (!operationId || !status)
|
|
88
|
+
return [];
|
|
89
|
+
return [{ operationId, status, ...(typeof receipt.detail === "string" ? { detail: receipt.detail } : typeof receipt.error === "string" ? { detail: receipt.error } : {}), ...(receipt.providerData !== undefined ? { providerData: receipt.providerData } : {}) }];
|
|
90
|
+
});
|
|
91
|
+
if (results.length === 0)
|
|
92
|
+
return undefined;
|
|
93
|
+
const timestamp = typeof updatedAt === "number" ? new Date(updatedAt).toISOString() : new Date().toISOString();
|
|
94
|
+
const provider = typeof row?.provider === "string" ? row.provider : "mock";
|
|
95
|
+
const status = row?.status === "applied" || row?.status === "partial" || row?.status === "failed" || row?.status === "rejected"
|
|
96
|
+
? row.status : results.every((result) => result.status === "applied" || result.status === "skipped") ? "applied" : "partial";
|
|
97
|
+
return {
|
|
98
|
+
planId, provider,
|
|
99
|
+
startedAt: typeof row?.startedAt === "string" ? row.startedAt : timestamp,
|
|
100
|
+
finishedAt: typeof row?.finishedAt === "string" ? row.finishedAt : timestamp,
|
|
101
|
+
status,
|
|
102
|
+
results,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
function failure(status) {
|
|
106
|
+
if (status === 409)
|
|
107
|
+
return { status: "conflict", reason: "hosted mirror has different immutable content" };
|
|
108
|
+
if (status === 401 || status === 403)
|
|
109
|
+
return { status: "unavailable", reason: "hosted authentication was rejected; pair the CLI again" };
|
|
110
|
+
return { status: "unavailable", reason: `hosted plan request failed (HTTP ${status})` };
|
|
111
|
+
}
|
|
112
|
+
export async function uploadHostedPatchPlan(stored, options = {}) {
|
|
113
|
+
const paired = broker();
|
|
114
|
+
if (!paired)
|
|
115
|
+
return { status: "unpaired" };
|
|
116
|
+
const document = hostedReviewDocument(stored.plan);
|
|
117
|
+
const digest = hostedPlanDigest(stored.plan);
|
|
118
|
+
const riskOrder = { low: 0, medium: 1, high: 2 };
|
|
119
|
+
const riskLevel = stored.plan.operations.reduce((risk, op) => riskOrder[op.riskLevel] > riskOrder[risk] ? op.riskLevel : risk, "low");
|
|
120
|
+
try {
|
|
121
|
+
const response = await (options.fetchImpl ?? fetch)(`${paired.baseUrl}${ENDPOINT}`, {
|
|
122
|
+
method: "POST",
|
|
123
|
+
headers: { Authorization: `Bearer ${paired.token}`, "Content-Type": "application/json" },
|
|
124
|
+
body: JSON.stringify({
|
|
125
|
+
externalPlanId: stored.plan.id, documentSha256: digest, externalRevision: stored.revision,
|
|
126
|
+
title: stored.plan.title, summary: stored.plan.summary, riskLevel, document,
|
|
127
|
+
operations: stored.plan.operations.map(mapOperation), createdAt: Date.parse(stored.plan.createdAt),
|
|
128
|
+
}),
|
|
129
|
+
signal: AbortSignal.timeout(options.timeoutMs ?? TIMEOUT_MS),
|
|
130
|
+
});
|
|
131
|
+
if (!response.ok)
|
|
132
|
+
return failure(response.status);
|
|
133
|
+
const raw = await response.json();
|
|
134
|
+
const state = parseState({ ...raw, externalPlanId: stored.plan.id, documentSha256: digest, approvedOperationIds: [], valueOverrides: {}, updatedAt: Date.now() }, paired.baseUrl);
|
|
135
|
+
return state ? { status: "saved", state } : { status: "unavailable", reason: "hosted plan response was invalid" };
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
return { status: "unavailable", reason: "hosted plan request was unavailable" };
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
export async function readHostedPatchPlan(planId, options = {}) {
|
|
142
|
+
const paired = broker();
|
|
143
|
+
if (!paired)
|
|
144
|
+
return { status: "unpaired" };
|
|
145
|
+
try {
|
|
146
|
+
const response = await (options.fetchImpl ?? fetch)(`${paired.baseUrl}${ENDPOINT}?externalPlanId=${encodeURIComponent(planId)}`, {
|
|
147
|
+
headers: { Authorization: `Bearer ${paired.token}`, Accept: "application/json" },
|
|
148
|
+
signal: AbortSignal.timeout(options.timeoutMs ?? TIMEOUT_MS),
|
|
149
|
+
});
|
|
150
|
+
if (response.status === 404)
|
|
151
|
+
return { status: "missing" };
|
|
152
|
+
if (!response.ok)
|
|
153
|
+
return failure(response.status);
|
|
154
|
+
const state = parseState(await response.json(), paired.baseUrl);
|
|
155
|
+
return state ? { status: "found", state } : { status: "unavailable", reason: "hosted plan response was invalid" };
|
|
156
|
+
}
|
|
157
|
+
catch {
|
|
158
|
+
return { status: "unavailable", reason: "hosted plan request was unavailable" };
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
/** Pull a peer's newer lifecycle into the local replica without weakening local integrity gates. */
|
|
162
|
+
export async function reconcileHostedPatchPlan(store, stored, options = {}) {
|
|
163
|
+
const hosted = await readHostedPatchPlan(stored.plan.id, options);
|
|
164
|
+
if (hosted.status === "unpaired" || hosted.status === "missing")
|
|
165
|
+
return { status: hosted.status };
|
|
166
|
+
if (hosted.status === "unavailable" || hosted.status === "conflict")
|
|
167
|
+
return hosted;
|
|
168
|
+
if (hosted.state.documentSha256 !== hostedPlanDigest(stored.plan)) {
|
|
169
|
+
return { status: "conflict", reason: "hosted review content does not match the immutable local plan" };
|
|
170
|
+
}
|
|
171
|
+
if (!["approved", "rejected", "applied"].includes(hosted.state.status))
|
|
172
|
+
return { status: "unchanged" };
|
|
173
|
+
try {
|
|
174
|
+
const reconciled = await store.reconcileReplica(stored.plan.id, {
|
|
175
|
+
status: hosted.state.status,
|
|
176
|
+
approvedOperationIds: hosted.state.approvedOperationIds,
|
|
177
|
+
valueOverrides: hosted.state.valueOverrides,
|
|
178
|
+
...(hosted.state.runRecord ? { run: hosted.state.runRecord } : {}),
|
|
179
|
+
});
|
|
180
|
+
return reconciled.changed
|
|
181
|
+
? { status: "updated", stored: reconciled.stored, remoteStatus: hosted.state.status }
|
|
182
|
+
: { status: "unchanged" };
|
|
183
|
+
}
|
|
184
|
+
catch (error) {
|
|
185
|
+
return { status: "conflict", reason: error instanceof Error ? error.message : "replica reconciliation failed" };
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
export async function reportHostedPlanLifecycle(stored, options = {}) {
|
|
189
|
+
const paired = broker();
|
|
190
|
+
if (!paired)
|
|
191
|
+
return { status: "unpaired" };
|
|
192
|
+
const status = stored.status === "applied" ? "applied" : stored.status === "rejected" ? "rejected" : "approved";
|
|
193
|
+
const runRecord = status === "applied" ? stored.runs.at(-1) : undefined;
|
|
194
|
+
const operationReceipts = runRecord?.results.map((result) => ({
|
|
195
|
+
packageOpId: result.operationId,
|
|
196
|
+
status: result.status,
|
|
197
|
+
...(result.detail ? { error: result.detail.slice(0, 1000) } : {}),
|
|
198
|
+
...(result.providerData !== undefined ? { providerData: result.providerData } : {}),
|
|
199
|
+
updatedAt: Date.parse(runRecord.finishedAt),
|
|
200
|
+
}));
|
|
201
|
+
try {
|
|
202
|
+
const response = await (options.fetchImpl ?? fetch)(`${paired.baseUrl}${ENDPOINT}`, {
|
|
203
|
+
method: "POST", headers: { Authorization: `Bearer ${paired.token}`, "Content-Type": "application/json" },
|
|
204
|
+
body: JSON.stringify({ action: "lifecycle", externalPlanId: stored.plan.id, documentSha256: hostedPlanDigest(stored.plan), status, approvedOperationIds: stored.approvedOperationIds, ...(options.claimId ? { claimId: options.claimId } : {}), ...(runRecord ? { runRecord, operationReceipts } : {}) }),
|
|
205
|
+
signal: AbortSignal.timeout(options.timeoutMs ?? TIMEOUT_MS),
|
|
206
|
+
});
|
|
207
|
+
if (!response.ok)
|
|
208
|
+
return failure(response.status);
|
|
209
|
+
const raw = await response.json();
|
|
210
|
+
const state = parseState({ ...raw, externalPlanId: stored.plan.id, documentSha256: hostedPlanDigest(stored.plan), approvedOperationIds: stored.approvedOperationIds, valueOverrides: stored.valueOverrides, updatedAt: Date.now() }, paired.baseUrl);
|
|
211
|
+
return state ? { status: "saved", state } : { status: "unavailable", reason: "hosted lifecycle response was invalid" };
|
|
212
|
+
}
|
|
213
|
+
catch {
|
|
214
|
+
return { status: "unavailable", reason: "hosted lifecycle request was unavailable" };
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
/** Coordinate online replicas before provider I/O; provider idempotency remains the offline fallback. */
|
|
218
|
+
export async function claimHostedPlanApply(stored, options = {}) {
|
|
219
|
+
const paired = broker();
|
|
220
|
+
if (!paired)
|
|
221
|
+
return { status: "unpaired" };
|
|
222
|
+
try {
|
|
223
|
+
const response = await (options.fetchImpl ?? fetch)(`${paired.baseUrl}${ENDPOINT}`, {
|
|
224
|
+
method: "POST", headers: { Authorization: `Bearer ${paired.token}`, "Content-Type": "application/json" },
|
|
225
|
+
body: JSON.stringify({ action: "claim", externalPlanId: stored.plan.id, documentSha256: hostedPlanDigest(stored.plan) }),
|
|
226
|
+
signal: AbortSignal.timeout(options.timeoutMs ?? TIMEOUT_MS),
|
|
227
|
+
});
|
|
228
|
+
if (!response.ok)
|
|
229
|
+
return failure(response.status);
|
|
230
|
+
const row = await response.json();
|
|
231
|
+
if (row.alreadyApplied === true || row.status === "applied")
|
|
232
|
+
return { status: "applied" };
|
|
233
|
+
if (row.status === "claimed" && typeof row.claimId === "string") {
|
|
234
|
+
return { status: "claimed", claimId: row.claimId, ...(typeof row.expiresAt === "number" ? { expiresAt: row.expiresAt } : {}) };
|
|
235
|
+
}
|
|
236
|
+
return { status: "unavailable", reason: "hosted apply claim response was invalid" };
|
|
237
|
+
}
|
|
238
|
+
catch {
|
|
239
|
+
return { status: "unavailable", reason: "hosted apply claim request was unavailable" };
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
/** Release a shared CLI claim only when preflight aborts before provider I/O. */
|
|
243
|
+
export async function releaseHostedPlanApply(stored, claimId, reason, options = {}) {
|
|
244
|
+
const paired = broker();
|
|
245
|
+
if (!paired)
|
|
246
|
+
return { status: "unpaired" };
|
|
247
|
+
try {
|
|
248
|
+
const response = await (options.fetchImpl ?? fetch)(`${paired.baseUrl}${ENDPOINT}`, {
|
|
249
|
+
method: "POST",
|
|
250
|
+
headers: { Authorization: `Bearer ${paired.token}`, "Content-Type": "application/json" },
|
|
251
|
+
body: JSON.stringify({
|
|
252
|
+
action: "release_claim",
|
|
253
|
+
externalPlanId: stored.plan.id,
|
|
254
|
+
documentSha256: hostedPlanDigest(stored.plan),
|
|
255
|
+
claimId,
|
|
256
|
+
reason,
|
|
257
|
+
}),
|
|
258
|
+
signal: AbortSignal.timeout(options.timeoutMs ?? TIMEOUT_MS),
|
|
259
|
+
});
|
|
260
|
+
if (!response.ok)
|
|
261
|
+
return failure(response.status);
|
|
262
|
+
const row = await response.json();
|
|
263
|
+
return row.released === true
|
|
264
|
+
? { status: "released" }
|
|
265
|
+
: { status: "unavailable", reason: "hosted apply claim release response was invalid" };
|
|
266
|
+
}
|
|
267
|
+
catch {
|
|
268
|
+
return { status: "unavailable", reason: "hosted apply claim release was unavailable" };
|
|
269
|
+
}
|
|
270
|
+
}
|
package/dist/icp.js
CHANGED
|
@@ -158,6 +158,14 @@ const CRUSTDATA_INDUSTRY = {
|
|
|
158
158
|
fintech: ["Financial Services"],
|
|
159
159
|
"financial services": ["Financial Services"],
|
|
160
160
|
};
|
|
161
|
+
// Common NAICS groups translated into Crustdata's LinkedIn industry vocabulary.
|
|
162
|
+
// This keeps an ICP authored with provider-neutral NAICS codes from silently
|
|
163
|
+
// dropping its industry constraint on the Pipe0 path.
|
|
164
|
+
const NAICS_CRUSTDATA_INDUSTRY = {
|
|
165
|
+
"5112": ["Software Development", "Computer Software"],
|
|
166
|
+
"5182": ["IT Services and IT Consulting", "Information Technology & Services"],
|
|
167
|
+
"5415": ["IT Services and IT Consulting", "Information Technology & Services"],
|
|
168
|
+
};
|
|
161
169
|
/**
|
|
162
170
|
* pipe0 Crustdata people-search config.filters from the ICP. `current_job_titles`
|
|
163
171
|
* matches real LinkedIn title strings (case-sensitive), so keywords are Title
|
|
@@ -185,9 +193,12 @@ export function icpToCrustdataFilters(icp) {
|
|
|
185
193
|
if (icp.firmographics.geos?.length) {
|
|
186
194
|
f.locations = icp.firmographics.geos.map((g) => COUNTRY_NAMES[g.toLowerCase()] ?? g);
|
|
187
195
|
}
|
|
188
|
-
if (icp.firmographics.industries?.length) {
|
|
196
|
+
if (icp.firmographics.industries?.length || icp.firmographics.naics?.length) {
|
|
189
197
|
const inds = [
|
|
190
|
-
...new Set(
|
|
198
|
+
...new Set([
|
|
199
|
+
...(icp.firmographics.industries ?? []).flatMap((i) => CRUSTDATA_INDUSTRY[i.toLowerCase()] ?? [titleCase(i)]),
|
|
200
|
+
...(icp.firmographics.naics ?? []).flatMap((code) => NAICS_CRUSTDATA_INDUSTRY[code] ?? []),
|
|
201
|
+
]),
|
|
191
202
|
];
|
|
192
203
|
if (inds.length)
|
|
193
204
|
f.current_employers_linkedin_industries = inds;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export { auditSnapshot, defaultPolicy } from "./audit.ts";
|
|
2
|
+
export { acquireCheckpointId, acquireCheckpointsDir, createFileAcquireCheckpointStore, validateAcquireCheckpoint, validateAcquireCheckpointKey, type AcquireCheckpoint, type AcquireCheckpointKey, type AcquireCheckpointStore, type AcquireContinuation, } from "./acquireCheckpoint.ts";
|
|
2
3
|
export { matchesTerritory, parseAssignmentPolicy, resolveAssignment, type AssignmentContext, type AssignmentPolicy, type AssignmentResult, type AssignmentStrategy, type TerritoryRule, } from "./assign.ts";
|
|
3
4
|
export { buildBulkUpdatePlan, isFilterableField, parseWhere, type BulkUpdateOptions } from "./bulkUpdate.ts";
|
|
4
5
|
export { buildDedupePlan, dedupeKey, type DedupeOptions } from "./dedupe.ts";
|
|
@@ -6,18 +7,20 @@ export { buildReassignPlans, type ReassignObjectType, type ReassignOptions } fro
|
|
|
6
7
|
export { buildLeadRoutePlan, type LeadRouteCounts, type LeadRouteOptions, type LeadRouteResult, } from "./route.ts";
|
|
7
8
|
export { accountHierarchyToMarkdown, buildAccountHierarchy, type AccountHierarchyConflict, type AccountHierarchyNode, type AccountHierarchyReport, } from "./hierarchy.ts";
|
|
8
9
|
export { buildRelationshipMap, relationshipMapToMarkdown, type RelationshipMap, type StakeholderNode, type StakeholderRole, type StakeholderSentiment, } from "./relationships.ts";
|
|
9
|
-
export { CONFIG_FILE_NAME, loadConfig, mergePolicy, resolveConfiguredRules, type FullstackgtmConfig, type LoadedConfig, } from "./config.ts";
|
|
10
|
+
export { CONFIG_FILE_NAME, loadConfig, mergePolicy, resolveConfiguredRules, rulePackageTrustFromCli, type FullstackgtmConfig, type LoadedConfig, type RulePackageTrust, } from "./config.ts";
|
|
10
11
|
export { applyPatchPlan, type ApplyPatchPlanOptions } from "./connector.ts";
|
|
11
12
|
export { APPLY_STAGES, BACKFILL_STRIPE_STAGES, composeListeners, createProgressEmitter, CRM_SYNC_STAGES, nullProgressEmitter, SNAPSHOT_PULL_STAGES, STRIPE_SNAPSHOT_STAGES, type ProgressEmitter, type ProgressEvent, type ProgressListener, type ProgressSnapshot, } from "./progress.ts";
|
|
12
13
|
export { createHubspotConnector, type HubspotConnectorOptions } from "./connectors/hubspot.ts";
|
|
13
14
|
export { DEFAULT_LOOPBACK_PORT, DEFAULT_OAUTH_SCOPES, exchangeHubspotCode, refreshHubspotToken, runHubspotLoopbackLogin, validateHubspotToken, type HubspotTokenSet, type LoopbackLoginOptions, } from "./connectors/hubspotAuth.ts";
|
|
14
15
|
export { createSalesforceConnector, type SalesforceConnection, type SalesforceConnectorOptions, } from "./connectors/salesforce.ts";
|
|
15
|
-
export { pollSalesforceDeviceLogin, refreshSalesforceToken, startSalesforceDeviceLogin, validateSalesforceToken, type SalesforceDeviceAuthorization, type SalesforceTokenSet, } from "./connectors/salesforceAuth.ts";
|
|
16
|
+
export { pollSalesforceDeviceLogin, refreshSalesforceToken, startSalesforceDeviceLogin, validateSalesforceOrigin, validateSalesforceToken, type SalesforceDeviceAuthorization, type SalesforceTokenSet, } from "./connectors/salesforceAuth.ts";
|
|
16
17
|
export { createStripeConnector, fetchStripePaidInvoices, type StripeConnectorOptions, type StripePaidInvoice, } from "./connectors/stripe.ts";
|
|
17
18
|
export { buildStripeBackfillPlan, DEFAULT_BACKFILL_MATCH_PROPERTY, type StripeBackfillCounts, type StripeBackfillOptions, type StripeBackfillResult, type StripeBackfillUnmatched, } from "./backfill.ts";
|
|
18
19
|
export { activeProfile, credentialsDir, credentialsPath, DEFAULT_PROFILE, deleteCredential, getCredential, listProfiles, resolveHubspotAccessToken, resolveHubspotConnection, setActiveProfile, storeCredential, type HubspotConnection, type StoredCredential, } from "./credentials.ts";
|
|
19
20
|
export { generateDemoSnapshot, type DemoSnapshotOptions } from "./demo.ts";
|
|
20
|
-
export { buildEnrichPlan, createFileEnrichRunStore, DEFAULT_STALE_DAYS, ENRICH_CONFIG_FILE_NAME, enrichRunId, inferIngestObjectType, ingestKeyValue, latestStamps, loadEnrichConfig, matchSourceRecord, parseCsv, parseEnrichConfig, resolveCrmField, selectStaleWork, sourceValueAt, stagedSourceRecords, staleDaysFor, type BuildEnrichPlanOptions, type EnrichAmbiguity, type EnrichConfig, type EnrichCounts, type EnrichFieldConfig, type EnrichMatchConfig, type EnrichMode, type EnrichObjectType, type EnrichPlanResult, type EnrichRun, type EnrichRunStore, type EnrichSourceConfig, type EnrichSourceRecord, type EnrichStamp, type EnrichWorkItem, type MatchOutcome, } from "./enrich.ts";
|
|
21
|
+
export { buildEnrichPlan, createFileEnrichRunStore, DEFAULT_STALE_DAYS, ENRICH_CONFIG_FILE_NAME, enrichRunId, inferIngestObjectType, ingestKeyValue, latestStamps, loadEnrichConfig, matchSourceRecord, parseCsv, parseEnrichConfig, resolveCrmField, selectStaleWork, sourceValueAt, stagedSourceRecords, staleDaysFor, type BuildEnrichPlanOptions, type AcquireDiscoveryCheckpoint, type EnrichAmbiguity, type EnrichConfig, type EnrichCounts, type EnrichFieldConfig, type EnrichMatchConfig, type EnrichMode, type EnrichObjectType, type EnrichPlanResult, type EnrichRun, type EnrichRunStore, type EnrichSourceConfig, type EnrichSourceRecord, type EnrichStamp, type EnrichWorkItem, type MatchOutcome, } from "./enrich.ts";
|
|
22
|
+
export { readHostedAcquireCheckpoint, writeHostedAcquireCheckpoint, type HostedAcquireCheckpoint, type HostedCheckpointReadResult, type HostedCheckpointWriteResult, } from "./hostedAcquireCheckpoint.ts";
|
|
23
|
+
export { claimHostedPlanApply, hostedPlanDigest, hostedReviewDocument, readHostedPatchPlan, reconcileHostedPatchPlan, releaseHostedPlanApply, reportHostedPlanLifecycle, uploadHostedPatchPlan } from "./hostedPatchPlan.ts";
|
|
21
24
|
export { scaffoldWorkspace, starterEnrichConfig, starterIcp, starterPlaybook, type InitProvider, type InitSource, type ScaffoldFile, type ScaffoldOptions, } from "./init.ts";
|
|
22
25
|
export { appendCoverage, classifyAccount, classifyCoverage, computeCoverage, coverageCountsFromSnapshot, coveredAccounts, coverageToText, crmCheckableCriteria, deriveAcvFromClosedWon, deriveBuyersPerAccount, estimateTam, loadTamModel, projectEta, readCoverageTimeline, saveTamModel, tamDir, tamReportToMarkdown, type AccountTamClass, type AcvBasis, type DerivedAcv, type DerivedBuyers, type EstimateTamInput, type TamClassified, type TamCoverage, type TamCoverageCounts, type TamCrossCheck, type TamEta, type TamModel, type TamTargeting, type TamUniverse, } from "./tam.ts";
|
|
23
26
|
export { apolloPullKeysForAppend, apolloPullKeysForRefresh, createApolloClient, pullApolloRecords, type ApolloClient, type ApolloClientOptions, type ApolloPullKey, type ApolloPullResult, } from "./enrichApollo.ts";
|
|
@@ -53,3 +56,4 @@ export { scoreBand, judgeRunId, normalizeSpan, isGroundedSpan, scoreAccount, acc
|
|
|
53
56
|
export { draft, authorOpeners, draftOpenerLlm, deterministicOpener, creditedTriggerSignal, draftEvidence, sanitizeOpener, firstLine, firstLineGroundedInTrigger, hasBannedGreeting, isStaleTrigger, draftOperationId, draftEvidenceId, DRAFT_CHANNELS, DRAFT_SCHEMA, DEFAULT_DRAFT_PROMPT, DEFAULT_FRESHNESS_DAYS, type Draft, type RejectedDraft, type DraftResult, type DraftChannel, } from "./draft.ts";
|
|
54
57
|
export { DEFAULT_GOLDEN_SET, DEFAULT_MIN_ACCURACY, HOT_SCORE, defaultJudgeFn, gradeAgainstOutcomes, gradeJudge, parseGoldenSet, type Confusion, type EvalJudgeFn, type GoldenHistory, type GoldenRow, type GradeResult, type OutcomeCalibration, } from "./judgeEval.ts";
|
|
55
58
|
export type { ApprovalStatus, AuditFinding, AuditFindingSeverity, CanonicalAccount, CanonicalActivity, CanonicalContact, CanonicalDeal, CanonicalGtmSnapshot, CanonicalUser, CrmProvider, GtmAuditRule, GtmConnector, GtmEvidence, GtmEvidenceSourceSystem, GtmObjectType, GtmPolicy, GtmRuleContext, GtmRuleResult, GtmSnapshotIndex, PatchOperation, PatchOperationResult, PatchOperationType, PatchPlan, PatchPlanAssumption, PatchPlanAssumptionConfidence, PatchPlanRun, PatchPlanRunStatus, PatchVerification, PipelineFinding, PipelineFindingStatus, PipelineFindingType, ProviderIdentity, RiskLevel, SourceFreshness, } from "./types.ts";
|
|
59
|
+
export { ProviderHttpError } from "./providerError.ts";
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export { auditSnapshot, defaultPolicy } from "./audit.js";
|
|
2
|
+
export { acquireCheckpointId, acquireCheckpointsDir, createFileAcquireCheckpointStore, validateAcquireCheckpoint, validateAcquireCheckpointKey, } from "./acquireCheckpoint.js";
|
|
2
3
|
export { matchesTerritory, parseAssignmentPolicy, resolveAssignment, } from "./assign.js";
|
|
3
4
|
export { buildBulkUpdatePlan, isFilterableField, parseWhere } from "./bulkUpdate.js";
|
|
4
5
|
export { buildDedupePlan, dedupeKey } from "./dedupe.js";
|
|
@@ -6,18 +7,20 @@ export { buildReassignPlans } from "./reassign.js";
|
|
|
6
7
|
export { buildLeadRoutePlan, } from "./route.js";
|
|
7
8
|
export { accountHierarchyToMarkdown, buildAccountHierarchy, } from "./hierarchy.js";
|
|
8
9
|
export { buildRelationshipMap, relationshipMapToMarkdown, } from "./relationships.js";
|
|
9
|
-
export { CONFIG_FILE_NAME, loadConfig, mergePolicy, resolveConfiguredRules, } from "./config.js";
|
|
10
|
+
export { CONFIG_FILE_NAME, loadConfig, mergePolicy, resolveConfiguredRules, rulePackageTrustFromCli, } from "./config.js";
|
|
10
11
|
export { applyPatchPlan } from "./connector.js";
|
|
11
12
|
export { APPLY_STAGES, BACKFILL_STRIPE_STAGES, composeListeners, createProgressEmitter, CRM_SYNC_STAGES, nullProgressEmitter, SNAPSHOT_PULL_STAGES, STRIPE_SNAPSHOT_STAGES, } from "./progress.js";
|
|
12
13
|
export { createHubspotConnector } from "./connectors/hubspot.js";
|
|
13
14
|
export { DEFAULT_LOOPBACK_PORT, DEFAULT_OAUTH_SCOPES, exchangeHubspotCode, refreshHubspotToken, runHubspotLoopbackLogin, validateHubspotToken, } from "./connectors/hubspotAuth.js";
|
|
14
15
|
export { createSalesforceConnector, } from "./connectors/salesforce.js";
|
|
15
|
-
export { pollSalesforceDeviceLogin, refreshSalesforceToken, startSalesforceDeviceLogin, validateSalesforceToken, } from "./connectors/salesforceAuth.js";
|
|
16
|
+
export { pollSalesforceDeviceLogin, refreshSalesforceToken, startSalesforceDeviceLogin, validateSalesforceOrigin, validateSalesforceToken, } from "./connectors/salesforceAuth.js";
|
|
16
17
|
export { createStripeConnector, fetchStripePaidInvoices, } from "./connectors/stripe.js";
|
|
17
18
|
export { buildStripeBackfillPlan, DEFAULT_BACKFILL_MATCH_PROPERTY, } from "./backfill.js";
|
|
18
19
|
export { activeProfile, credentialsDir, credentialsPath, DEFAULT_PROFILE, deleteCredential, getCredential, listProfiles, resolveHubspotAccessToken, resolveHubspotConnection, setActiveProfile, storeCredential, } from "./credentials.js";
|
|
19
20
|
export { generateDemoSnapshot } from "./demo.js";
|
|
20
21
|
export { buildEnrichPlan, createFileEnrichRunStore, DEFAULT_STALE_DAYS, ENRICH_CONFIG_FILE_NAME, enrichRunId, inferIngestObjectType, ingestKeyValue, latestStamps, loadEnrichConfig, matchSourceRecord, parseCsv, parseEnrichConfig, resolveCrmField, selectStaleWork, sourceValueAt, stagedSourceRecords, staleDaysFor, } from "./enrich.js";
|
|
22
|
+
export { readHostedAcquireCheckpoint, writeHostedAcquireCheckpoint, } from "./hostedAcquireCheckpoint.js";
|
|
23
|
+
export { claimHostedPlanApply, hostedPlanDigest, hostedReviewDocument, readHostedPatchPlan, reconcileHostedPatchPlan, releaseHostedPlanApply, reportHostedPlanLifecycle, uploadHostedPatchPlan } from "./hostedPatchPlan.js";
|
|
21
24
|
export { scaffoldWorkspace, starterEnrichConfig, starterIcp, starterPlaybook, } from "./init.js";
|
|
22
25
|
export { appendCoverage, classifyAccount, classifyCoverage, computeCoverage, coverageCountsFromSnapshot, coveredAccounts, coverageToText, crmCheckableCriteria, deriveAcvFromClosedWon, deriveBuyersPerAccount, estimateTam, loadTamModel, projectEta, readCoverageTimeline, saveTamModel, tamDir, tamReportToMarkdown, } from "./tam.js";
|
|
23
26
|
export { apolloPullKeysForAppend, apolloPullKeysForRefresh, createApolloClient, pullApolloRecords, } from "./enrichApollo.js";
|
|
@@ -52,3 +55,4 @@ export { fetchAtsJobs, snippetFor, } from "./connectors/atsBoards.js";
|
|
|
52
55
|
export { scoreBand, judgeRunId, normalizeSpan, isGroundedSpan, scoreAccount, accountRecentlyTouched, bestContactForAccount, deterministicDecision, deterministicWhyNow, JUDGE_SCHEMA, DEFAULT_JUDGE_PROMPT, judgeDecisionLlm, judgeSignals, judgeDir, createFileJudgeStore, } from "./judge.js";
|
|
53
56
|
export { draft, authorOpeners, draftOpenerLlm, deterministicOpener, creditedTriggerSignal, draftEvidence, sanitizeOpener, firstLine, firstLineGroundedInTrigger, hasBannedGreeting, isStaleTrigger, draftOperationId, draftEvidenceId, DRAFT_CHANNELS, DRAFT_SCHEMA, DEFAULT_DRAFT_PROMPT, DEFAULT_FRESHNESS_DAYS, } from "./draft.js";
|
|
54
57
|
export { DEFAULT_GOLDEN_SET, DEFAULT_MIN_ACCURACY, HOT_SCORE, defaultJudgeFn, gradeAgainstOutcomes, gradeJudge, parseGoldenSet, } from "./judgeEval.js";
|
|
58
|
+
export { ProviderHttpError } from "./providerError.js";
|
package/dist/market.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type ProgressEmitter } from "./progress.ts";
|
|
2
2
|
import type { GtmEvidence } from "./types.ts";
|
|
3
|
+
export { assertPublicUrl } from "./publicHttp.ts";
|
|
3
4
|
/**
|
|
4
5
|
* The Market Map: a live model of the competitive category a company sells
|
|
5
6
|
* into. Vendors publish claims constantly (pricing pages, feature pages,
|
|
@@ -162,7 +163,6 @@ export type FetchPage = (url: string) => Promise<{
|
|
|
162
163
|
status: number;
|
|
163
164
|
body: string;
|
|
164
165
|
}>;
|
|
165
|
-
export declare function assertPublicUrl(rawUrl: string): Promise<URL>;
|
|
166
166
|
export type CaptureOptions = {
|
|
167
167
|
/** Directory for captures; defaults to <marketHome>/captures. Ignored when `store` is given. */
|
|
168
168
|
dir?: string;
|
package/dist/market.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { lookup } from "node:dns/promises";
|
|
3
2
|
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
|
4
|
-
import { isIP } from "node:net";
|
|
5
3
|
import { join } from "node:path";
|
|
6
4
|
import { credentialsDir } from "./credentials.js";
|
|
7
5
|
import { MARKET_CAPTURE_STAGES, nullProgressEmitter } from "./progress.js";
|
|
6
|
+
import { publicHttpGet } from "./publicHttp.js";
|
|
7
|
+
export { assertPublicUrl } from "./publicHttp.js";
|
|
8
8
|
const INTENSITY_RANK = {
|
|
9
9
|
loud: 3,
|
|
10
10
|
quiet: 2,
|
|
@@ -152,136 +152,16 @@ export function extractReadableText(html) {
|
|
|
152
152
|
* any host that is or resolves to a private/loopback/link-local/metadata
|
|
153
153
|
* address, and (3) follow redirects manually, re-validating each hop.
|
|
154
154
|
*
|
|
155
|
-
*
|
|
156
|
-
*
|
|
157
|
-
* competitor pages; a hardened deployment should fetch through an egress proxy.
|
|
155
|
+
* The default transport resolves once, rejects mixed public/private answers,
|
|
156
|
+
* and pins a validated address into socket creation to prevent DNS rebinding.
|
|
158
157
|
*/
|
|
159
158
|
const MAX_REDIRECTS = 5;
|
|
160
159
|
const FETCH_TIMEOUT_MS = 15_000;
|
|
161
160
|
const MAX_BODY_BYTES = 5_000_000;
|
|
162
|
-
function ipv4IsPrivate(ip) {
|
|
163
|
-
const parts = ip.split(".").map((n) => Number(n));
|
|
164
|
-
if (parts.length !== 4 || parts.some((n) => !Number.isInteger(n) || n < 0 || n > 255))
|
|
165
|
-
return true;
|
|
166
|
-
const [a, b] = parts;
|
|
167
|
-
if (a === 0 || a === 127)
|
|
168
|
-
return true; // this-host, loopback
|
|
169
|
-
if (a === 10)
|
|
170
|
-
return true; // private
|
|
171
|
-
if (a === 172 && b >= 16 && b <= 31)
|
|
172
|
-
return true; // private
|
|
173
|
-
if (a === 192 && b === 168)
|
|
174
|
-
return true; // private
|
|
175
|
-
if (a === 169 && b === 254)
|
|
176
|
-
return true; // link-local incl. 169.254.169.254 metadata
|
|
177
|
-
if (a === 100 && b >= 64 && b <= 127)
|
|
178
|
-
return true; // CGNAT
|
|
179
|
-
if (a >= 224)
|
|
180
|
-
return true; // multicast / reserved
|
|
181
|
-
return false;
|
|
182
|
-
}
|
|
183
|
-
function ipIsPrivate(ip) {
|
|
184
|
-
const family = isIP(ip);
|
|
185
|
-
if (family === 4)
|
|
186
|
-
return ipv4IsPrivate(ip);
|
|
187
|
-
if (family === 6) {
|
|
188
|
-
const lower = ip.toLowerCase();
|
|
189
|
-
if (lower === "::1" || lower === "::")
|
|
190
|
-
return true; // loopback / unspecified
|
|
191
|
-
// IPv4-mapped (::ffff:…) — Node normalizes ::ffff:127.0.0.1 to ::ffff:7f00:1,
|
|
192
|
-
// so accept both the dotted and the hex-pair forms, unwrap, check the v4.
|
|
193
|
-
const mapped = lower.match(/^::ffff:(.+)$/);
|
|
194
|
-
if (mapped) {
|
|
195
|
-
const rest = mapped[1];
|
|
196
|
-
if (rest.includes("."))
|
|
197
|
-
return ipv4IsPrivate(rest);
|
|
198
|
-
const groups = rest.split(":");
|
|
199
|
-
if (groups.length === 2) {
|
|
200
|
-
const hi = parseInt(groups[0], 16);
|
|
201
|
-
const lo = parseInt(groups[1], 16);
|
|
202
|
-
if (Number.isNaN(hi) || Number.isNaN(lo))
|
|
203
|
-
return true;
|
|
204
|
-
return ipv4IsPrivate(`${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`);
|
|
205
|
-
}
|
|
206
|
-
return true; // unrecognized mapped form → refuse
|
|
207
|
-
}
|
|
208
|
-
if (lower.startsWith("fe8") || lower.startsWith("fe9") || lower.startsWith("fea") || lower.startsWith("feb"))
|
|
209
|
-
return true; // link-local fe80::/10
|
|
210
|
-
if (lower.startsWith("fc") || lower.startsWith("fd"))
|
|
211
|
-
return true; // unique-local fc00::/7
|
|
212
|
-
return false;
|
|
213
|
-
}
|
|
214
|
-
return true; // not a recognizable IP literal → refuse
|
|
215
|
-
}
|
|
216
|
-
export async function assertPublicUrl(rawUrl) {
|
|
217
|
-
let url;
|
|
218
|
-
try {
|
|
219
|
-
url = new URL(rawUrl);
|
|
220
|
-
}
|
|
221
|
-
catch {
|
|
222
|
-
throw new Error(`market capture: "${rawUrl}" is not a valid URL.`);
|
|
223
|
-
}
|
|
224
|
-
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
225
|
-
throw new Error(`market capture refuses ${url.protocol} URLs (only http/https): ${rawUrl}`);
|
|
226
|
-
}
|
|
227
|
-
const host = url.hostname.replace(/^\[|\]$/g, ""); // strip IPv6 brackets
|
|
228
|
-
if (isIP(host)) {
|
|
229
|
-
if (ipIsPrivate(host))
|
|
230
|
-
throw new Error(`market capture refuses private/loopback address ${host} (SSRF guard).`);
|
|
231
|
-
return url;
|
|
232
|
-
}
|
|
233
|
-
// Hostname: resolve and refuse if ANY address is private.
|
|
234
|
-
const addrs = await lookup(host, { all: true });
|
|
235
|
-
for (const { address } of addrs) {
|
|
236
|
-
if (ipIsPrivate(address)) {
|
|
237
|
-
throw new Error(`market capture refuses ${host} — it resolves to private/internal address ${address} (SSRF guard).`);
|
|
238
|
-
}
|
|
239
|
-
}
|
|
240
|
-
return url;
|
|
241
|
-
}
|
|
242
161
|
const defaultFetchPage = async (url) => {
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
const controller = new AbortController();
|
|
247
|
-
const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
|
|
248
|
-
let response;
|
|
249
|
-
try {
|
|
250
|
-
response = await fetch(current, {
|
|
251
|
-
headers: {
|
|
252
|
-
"User-Agent": "fullstackgtm-market/0 (+https://github.com/fullstackgtm/core)",
|
|
253
|
-
"Accept-Language": "en-US",
|
|
254
|
-
},
|
|
255
|
-
redirect: "manual",
|
|
256
|
-
signal: controller.signal,
|
|
257
|
-
});
|
|
258
|
-
}
|
|
259
|
-
finally {
|
|
260
|
-
clearTimeout(timer);
|
|
261
|
-
}
|
|
262
|
-
if (response.status >= 300 && response.status < 400 && response.headers.get("location")) {
|
|
263
|
-
current = new URL(response.headers.get("location"), current).toString();
|
|
264
|
-
continue; // re-validate the redirect target on the next iteration
|
|
265
|
-
}
|
|
266
|
-
const reader = response.body?.getReader();
|
|
267
|
-
if (!reader)
|
|
268
|
-
return { status: response.status, body: await response.text() };
|
|
269
|
-
const chunks = [];
|
|
270
|
-
let total = 0;
|
|
271
|
-
for (;;) {
|
|
272
|
-
const { done, value } = await reader.read();
|
|
273
|
-
if (done)
|
|
274
|
-
break;
|
|
275
|
-
total += value.length;
|
|
276
|
-
if (total > MAX_BODY_BYTES) {
|
|
277
|
-
await reader.cancel();
|
|
278
|
-
break;
|
|
279
|
-
}
|
|
280
|
-
chunks.push(value);
|
|
281
|
-
}
|
|
282
|
-
return { status: response.status, body: Buffer.concat(chunks).toString("utf8") };
|
|
283
|
-
}
|
|
284
|
-
throw new Error(`market capture: too many redirects (>${MAX_REDIRECTS}) for ${url}`);
|
|
162
|
+
const response = await publicHttpGet(url, { maxBytes: MAX_BODY_BYTES, maxRedirects: MAX_REDIRECTS,
|
|
163
|
+
timeoutMs: FETCH_TIMEOUT_MS, headers: { "User-Agent": "fullstackgtm-market/0 (+https://github.com/fullstackgtm/core)", "Accept-Language": "en-US" } });
|
|
164
|
+
return { status: response.status, body: Buffer.from(response.body).toString("utf8") };
|
|
285
165
|
};
|
|
286
166
|
export async function captureMarket(config, options = {}) {
|
|
287
167
|
const store = options.store ?? createFileMarketStore(config.category, { capturesDir: options.dir });
|
package/dist/marketClassify.d.ts
CHANGED
|
@@ -51,6 +51,16 @@ export type MarketWorksheet = {
|
|
|
51
51
|
captureHash: string;
|
|
52
52
|
text: string;
|
|
53
53
|
}>;
|
|
54
|
+
/** Deterministic retrieval hints: page lines that contain claim terms. These are not labels. */
|
|
55
|
+
claimHints: Array<{
|
|
56
|
+
claimId: string;
|
|
57
|
+
matches: Array<{
|
|
58
|
+
url: string;
|
|
59
|
+
captureHash: string;
|
|
60
|
+
term: string;
|
|
61
|
+
quote: string;
|
|
62
|
+
}>;
|
|
63
|
+
}>;
|
|
54
64
|
instructions: string;
|
|
55
65
|
};
|
|
56
66
|
export declare function buildWorksheet(config: MarketConfig, vendorId: string, options?: {
|
package/dist/marketClassify.js
CHANGED
|
@@ -188,6 +188,56 @@ export async function classifyMarket(config, options) {
|
|
|
188
188
|
retriedVendorIds,
|
|
189
189
|
};
|
|
190
190
|
}
|
|
191
|
+
const HINT_STOPWORDS = new Set([
|
|
192
|
+
"the", "and", "for", "with", "from", "that", "this", "into", "your", "you", "are", "but", "not", "when", "how", "use", "uses", "using",
|
|
193
|
+
"claim", "claims", "vendor", "vendors", "page", "pages", "loud", "quiet", "absent", "capability", "specific", "differentiating",
|
|
194
|
+
"unknown", "buyer", "buyers", "pricing", "structure", "category", "support", "supported", "primary", "secondary", "mentioned", "mentions",
|
|
195
|
+
]);
|
|
196
|
+
function normalizeHintText(value) {
|
|
197
|
+
return value.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim().replace(/\s+/g, " ");
|
|
198
|
+
}
|
|
199
|
+
function claimHintTerms(claim) {
|
|
200
|
+
const exact = [claim.id.replace(/[-_]+/g, " "), claim.capability, ...(claim.terms ?? [])]
|
|
201
|
+
.map(normalizeHintText)
|
|
202
|
+
.filter((term) => term.length >= 3);
|
|
203
|
+
const tokenSource = normalizeHintText(`${claim.capability} ${claim.definition} ${(claim.terms ?? []).join(" ")}`)
|
|
204
|
+
.split(" ")
|
|
205
|
+
.filter((term) => term.length >= 4 && !HINT_STOPWORDS.has(term));
|
|
206
|
+
return Array.from(new Set([...exact, ...tokenSource])).slice(0, 24);
|
|
207
|
+
}
|
|
208
|
+
function lineSnippet(line, term) {
|
|
209
|
+
const clean = line.trim().replace(/\s+/g, " ");
|
|
210
|
+
if (clean.length <= 300)
|
|
211
|
+
return clean;
|
|
212
|
+
const lower = clean.toLowerCase();
|
|
213
|
+
const idx = lower.indexOf(term.toLowerCase().split(" ")[0] ?? "");
|
|
214
|
+
const start = Math.max(0, idx < 0 ? 0 : idx - 120);
|
|
215
|
+
return clean.slice(start, start + 300).trim();
|
|
216
|
+
}
|
|
217
|
+
function buildClaimHints(claims, pages) {
|
|
218
|
+
return claims.map((claim) => {
|
|
219
|
+
const terms = claimHintTerms(claim);
|
|
220
|
+
const matches = [];
|
|
221
|
+
for (const page of pages) {
|
|
222
|
+
const lines = page.text.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
|
|
223
|
+
for (const term of terms) {
|
|
224
|
+
const normalizedTerm = normalizeHintText(term);
|
|
225
|
+
const line = lines.find((candidate) => normalizeHintText(candidate).includes(normalizedTerm));
|
|
226
|
+
if (!line)
|
|
227
|
+
continue;
|
|
228
|
+
const quote = lineSnippet(line, term);
|
|
229
|
+
if (quote && !matches.some((m) => m.quote === quote && m.url === page.url)) {
|
|
230
|
+
matches.push({ url: page.url, captureHash: page.captureHash, term, quote });
|
|
231
|
+
}
|
|
232
|
+
if (matches.length >= 4)
|
|
233
|
+
break;
|
|
234
|
+
}
|
|
235
|
+
if (matches.length >= 4)
|
|
236
|
+
break;
|
|
237
|
+
}
|
|
238
|
+
return { claimId: claim.id, matches };
|
|
239
|
+
});
|
|
240
|
+
}
|
|
191
241
|
export function buildWorksheet(config, vendorId, options = {}) {
|
|
192
242
|
const vendor = config.vendors.find((candidate) => candidate.id === vendorId);
|
|
193
243
|
if (!vendor)
|
|
@@ -211,6 +261,7 @@ export function buildWorksheet(config, vendorId, options = {}) {
|
|
|
211
261
|
vendor: { id: vendor.id, name: vendor.name },
|
|
212
262
|
claims: config.claims,
|
|
213
263
|
pages,
|
|
214
|
-
|
|
264
|
+
claimHints: buildClaimHints(config.claims, pages),
|
|
265
|
+
instructions: "Produce one observation per claim from these pages only. Intensity rubric: loud = hero copy, primary positioning, or a named/dedicated product area; quiet = supported by a verbatim page quote but secondary, qualified, or merely available; absent = the pages do not support the claim; unobservable = no usable captured page exists for this vendor, not a synonym for absent. claimHints are deterministic retrieval hints only: use them to find candidate support, but do not mark a claim loud/quiet unless the quote truly supports the claim. Every loud/quiet reading must quote a verbatim span (≤300 chars) from a page's text, with that page's url and captureHash in evidence metadata. Submit as an ObservationSet via `market observe --from <file>` — quotes are mechanically verified against the captures.",
|
|
215
266
|
};
|
|
216
267
|
}
|