@graphext/cuery 0.4.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/browser.d.ts +1 -1
- package/esm/browser.d.ts.map +1 -1
- package/esm/browser.js +1 -1
- package/esm/mod.d.ts +5 -2
- package/esm/mod.d.ts.map +1 -1
- package/esm/mod.js +7 -2
- package/esm/src/api.d.ts +36 -5
- package/esm/src/api.d.ts.map +1 -1
- package/esm/src/api.js +84 -37
- package/esm/src/apis/chatgptScraper/brightdata.js +1 -1
- package/esm/src/apis/chatgptScraper/oxy.js +1 -1
- package/esm/src/apis/chatgptScraper/scraper.js +2 -2
- package/esm/src/apis/hasdata/aim.js +1 -1
- package/esm/src/apis/hasdata/aio.js +1 -1
- package/esm/src/apis/hasdata/helpers.d.ts +1 -1
- package/esm/src/apis/hasdata/helpers.d.ts.map +1 -1
- package/esm/src/apis/hasdata/helpers.js +2 -2
- package/esm/src/assets/models.d.ts +60725 -0
- package/esm/src/assets/models.d.ts.map +1 -0
- package/esm/src/assets/models.js +71915 -0
- package/esm/src/helpers/async.d.ts.map +1 -0
- package/esm/src/{async.js → helpers/async.js} +1 -1
- package/esm/src/helpers/seedKeywords.d.ts.map +1 -0
- package/esm/src/helpers/urls.d.ts.map +1 -0
- package/esm/src/helpers/utils.d.ts.map +1 -0
- package/esm/src/llm.d.ts +35 -0
- package/esm/src/llm.d.ts.map +1 -0
- package/esm/src/llm.js +59 -0
- package/esm/src/providers/google.d.ts +12 -0
- package/esm/src/providers/google.d.ts.map +1 -0
- package/esm/src/providers/google.js +111 -0
- package/esm/src/providers/index.d.ts +13 -0
- package/esm/src/providers/index.d.ts.map +1 -0
- package/esm/src/providers/index.js +14 -0
- package/esm/src/providers/openai.d.ts +12 -0
- package/esm/src/providers/openai.d.ts.map +1 -0
- package/esm/src/providers/openai.js +141 -0
- package/esm/src/providers/pricing.d.ts +72 -0
- package/esm/src/providers/pricing.d.ts.map +1 -0
- package/esm/src/providers/pricing.js +88 -0
- package/esm/src/providers/registry.d.ts +20 -0
- package/esm/src/providers/registry.d.ts.map +1 -0
- package/esm/src/providers/registry.js +35 -0
- package/esm/src/providers/types.d.ts +49 -0
- package/esm/src/providers/types.d.ts.map +1 -0
- package/esm/src/providers/types.js +7 -0
- package/esm/src/response.d.ts +74 -0
- package/esm/src/response.d.ts.map +1 -0
- package/esm/src/response.js +110 -0
- package/esm/src/tool.d.ts +58 -0
- package/esm/src/tool.d.ts.map +1 -0
- package/esm/src/tool.js +91 -0
- package/esm/src/tools/brands.js +6 -6
- package/esm/src/tools/classifier.d.ts +80 -17
- package/esm/src/tools/classifier.d.ts.map +1 -1
- package/esm/src/tools/classifier.js +68 -80
- package/esm/src/tools/entities.d.ts +23 -12
- package/esm/src/tools/entities.d.ts.map +1 -1
- package/esm/src/tools/entities.js +27 -47
- package/esm/src/tools/funnel.js +7 -7
- package/esm/src/tools/generic.d.ts +17 -4
- package/esm/src/tools/generic.d.ts.map +1 -1
- package/esm/src/tools/generic.js +39 -14
- package/esm/src/tools/keywords.js +5 -5
- package/esm/src/tools/personas.d.ts +49 -2
- package/esm/src/tools/personas.d.ts.map +1 -1
- package/esm/src/tools/personas.js +59 -35
- package/esm/src/tools/scorer.d.ts +24 -6
- package/esm/src/tools/scorer.d.ts.map +1 -1
- package/esm/src/tools/scorer.js +27 -22
- package/esm/src/tools/search.d.ts.map +1 -1
- package/esm/src/tools/search.js +33 -9
- package/esm/src/tools/sentiment.d.ts +30 -8
- package/esm/src/tools/sentiment.d.ts.map +1 -1
- package/esm/src/tools/sentiment.js +33 -28
- package/esm/src/tools/sources.d.ts +5 -5
- package/esm/src/tools/sources.d.ts.map +1 -1
- package/esm/src/tools/sources.js +5 -6
- package/esm/src/tools/topics.d.ts +44 -16
- package/esm/src/tools/topics.d.ts.map +1 -1
- package/esm/src/tools/topics.js +77 -68
- package/esm/src/tools/translate.d.ts +22 -31
- package/esm/src/tools/translate.d.ts.map +1 -1
- package/esm/src/tools/translate.js +40 -36
- package/package.json +2 -1
- package/script/browser.d.ts +1 -1
- package/script/browser.d.ts.map +1 -1
- package/script/browser.js +1 -1
- package/script/mod.d.ts +5 -2
- package/script/mod.d.ts.map +1 -1
- package/script/mod.js +14 -2
- package/script/src/api.d.ts +36 -5
- package/script/src/api.d.ts.map +1 -1
- package/script/src/api.js +84 -35
- package/script/src/apis/chatgptScraper/brightdata.js +1 -1
- package/script/src/apis/chatgptScraper/oxy.js +1 -1
- package/script/src/apis/chatgptScraper/scraper.js +2 -2
- package/script/src/apis/hasdata/aim.js +1 -1
- package/script/src/apis/hasdata/aio.js +1 -1
- package/script/src/apis/hasdata/helpers.d.ts +1 -1
- package/script/src/apis/hasdata/helpers.d.ts.map +1 -1
- package/script/src/apis/hasdata/helpers.js +2 -2
- package/script/src/assets/models.d.ts +60725 -0
- package/script/src/assets/models.d.ts.map +1 -0
- package/script/src/assets/models.js +71917 -0
- package/script/src/helpers/async.d.ts.map +1 -0
- package/script/src/{async.js → helpers/async.js} +1 -1
- package/script/src/helpers/seedKeywords.d.ts.map +1 -0
- package/script/src/helpers/urls.d.ts.map +1 -0
- package/script/src/helpers/utils.d.ts.map +1 -0
- package/script/src/llm.d.ts +35 -0
- package/script/src/llm.d.ts.map +1 -0
- package/script/src/llm.js +65 -0
- package/script/src/providers/google.d.ts +12 -0
- package/script/src/providers/google.d.ts.map +1 -0
- package/script/src/providers/google.js +148 -0
- package/script/src/providers/index.d.ts +13 -0
- package/script/src/providers/index.d.ts.map +1 -0
- package/script/src/providers/index.js +24 -0
- package/script/src/providers/openai.d.ts +12 -0
- package/script/src/providers/openai.d.ts.map +1 -0
- package/script/src/providers/openai.js +181 -0
- package/script/src/providers/pricing.d.ts +72 -0
- package/script/src/providers/pricing.d.ts.map +1 -0
- package/script/src/providers/pricing.js +97 -0
- package/script/src/providers/registry.d.ts +20 -0
- package/script/src/providers/registry.d.ts.map +1 -0
- package/script/src/providers/registry.js +39 -0
- package/script/src/providers/types.d.ts +49 -0
- package/script/src/providers/types.d.ts.map +1 -0
- package/script/src/providers/types.js +8 -0
- package/script/src/response.d.ts +74 -0
- package/script/src/response.d.ts.map +1 -0
- package/script/src/response.js +114 -0
- package/script/src/tool.d.ts +58 -0
- package/script/src/tool.d.ts.map +1 -0
- package/script/src/tool.js +95 -0
- package/script/src/tools/brands.js +6 -6
- package/script/src/tools/classifier.d.ts +80 -17
- package/script/src/tools/classifier.d.ts.map +1 -1
- package/script/src/tools/classifier.js +72 -85
- package/script/src/tools/entities.d.ts +23 -12
- package/script/src/tools/entities.d.ts.map +1 -1
- package/script/src/tools/entities.js +29 -51
- package/script/src/tools/funnel.js +7 -7
- package/script/src/tools/generic.d.ts +17 -4
- package/script/src/tools/generic.d.ts.map +1 -1
- package/script/src/tools/generic.js +39 -14
- package/script/src/tools/keywords.js +5 -5
- package/script/src/tools/personas.d.ts +49 -2
- package/script/src/tools/personas.d.ts.map +1 -1
- package/script/src/tools/personas.js +63 -36
- package/script/src/tools/scorer.d.ts +24 -6
- package/script/src/tools/scorer.d.ts.map +1 -1
- package/script/src/tools/scorer.js +28 -24
- package/script/src/tools/search.d.ts.map +1 -1
- package/script/src/tools/search.js +69 -9
- package/script/src/tools/sentiment.d.ts +30 -8
- package/script/src/tools/sentiment.d.ts.map +1 -1
- package/script/src/tools/sentiment.js +37 -30
- package/script/src/tools/sources.d.ts +5 -5
- package/script/src/tools/sources.d.ts.map +1 -1
- package/script/src/tools/sources.js +4 -5
- package/script/src/tools/topics.d.ts +44 -16
- package/script/src/tools/topics.d.ts.map +1 -1
- package/script/src/tools/topics.js +80 -72
- package/script/src/tools/translate.d.ts +22 -31
- package/script/src/tools/translate.d.ts.map +1 -1
- package/script/src/tools/translate.js +43 -40
- package/esm/src/async.d.ts.map +0 -1
- package/esm/src/models.d.ts +0 -18
- package/esm/src/models.d.ts.map +0 -1
- package/esm/src/models.js +0 -48
- package/esm/src/openai.d.ts +0 -17
- package/esm/src/openai.d.ts.map +0 -1
- package/esm/src/openai.js +0 -136
- package/esm/src/tools/seedKeywords.d.ts.map +0 -1
- package/esm/src/urls.d.ts.map +0 -1
- package/esm/src/utils.d.ts.map +0 -1
- package/script/src/async.d.ts.map +0 -1
- package/script/src/models.d.ts +0 -18
- package/script/src/models.d.ts.map +0 -1
- package/script/src/models.js +0 -52
- package/script/src/openai.d.ts +0 -17
- package/script/src/openai.d.ts.map +0 -1
- package/script/src/openai.js +0 -175
- package/script/src/tools/seedKeywords.d.ts.map +0 -1
- package/script/src/urls.d.ts.map +0 -1
- package/script/src/utils.d.ts.map +0 -1
- /package/esm/src/{async.d.ts → helpers/async.d.ts} +0 -0
- /package/esm/src/{tools → helpers}/seedKeywords.d.ts +0 -0
- /package/esm/src/{tools → helpers}/seedKeywords.js +0 -0
- /package/esm/src/{urls.d.ts → helpers/urls.d.ts} +0 -0
- /package/esm/src/{urls.js → helpers/urls.js} +0 -0
- /package/esm/src/{utils.d.ts → helpers/utils.d.ts} +0 -0
- /package/esm/src/{utils.js → helpers/utils.js} +0 -0
- /package/script/src/{async.d.ts → helpers/async.d.ts} +0 -0
- /package/script/src/{tools → helpers}/seedKeywords.d.ts +0 -0
- /package/script/src/{tools → helpers}/seedKeywords.js +0 -0
- /package/script/src/{urls.d.ts → helpers/urls.d.ts} +0 -0
- /package/script/src/{urls.js → helpers/urls.js} +0 -0
- /package/script/src/{utils.d.ts → helpers/utils.d.ts} +0 -0
- /package/script/src/{utils.js → helpers/utils.js} +0 -0
package/script/src/api.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../../src/src/api.ts"],"names":[],"mappings":"AAGA,OAAO,EAAwC,KAAK,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAIlG,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../../src/src/api.ts"],"names":[],"mappings":"AAGA,OAAO,EAAwC,KAAK,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAIlG,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAElE,OAAO,EAAE,KAAK,OAAO,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAE9D,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,cAAc,EAAE,KAAK,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAMlG,MAAM,MAAM,aAAa,GAAG;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7B,aAAa,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC;IAC/B,MAAM,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,oBAAoB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACpC,0BAA0B,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC1C,2BAA2B,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CAC3C,CAAC;AAEF,wBAAsB,UAAU,CAC/B,OAAO,EAAE,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,EAC9B,KAAK,EAAE,eAAe,EACtB,aAAa,GAAE,MAAM,GAAG,IAAW,GACjC,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAkD/B;AAQD,wBAAsB,cAAc,CACnC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,GAClB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAc/B;AAED,wBAAsB,aAAa,CAAC,MAAM,EAAE;IAC3C,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IACxC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CACf;;;;;UAWA;AAED,wBAAsB,gBAAgB,CAAC,MAAM,EAAE;IAC9C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;CACf;;;;;;;UAiBA;AAED,wBAAsB,sBAAsB,CAC3C,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,GAClB,OAAO,CAAC,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAAC,CA0BnE;AAED,wBAAsB,kBAAkB,CACvC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EACpB,MAAM,EAAE,MAAM,GACZ,OAAO,CAAC,KAAK,CAAC;IAAE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAAC,CAa/E;AAED,wBAAsB,oBAAoB,CACzC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EACpB,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,GACtB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAkBtC;AAED,wBAAsB,yBAAyB,CAC9C,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,GAClB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAkB/B;AAED,wBAAsB,eAAe,CACpC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,GAClB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAetC;AAED,wBAAsB,wBAAwB,CAC7C,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EACpB,OAAO,GAAE,MAAW,GAClB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAe/B;AAED,wBAAsB,cAAc,CACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,EACtB,OAAO,EAAE,YAAY,GACnB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CA6C/B"}
|
package/script/src/api.js
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.queryModel = queryModel;
|
|
4
4
|
exports.classifyIntent = classifyIntent;
|
|
5
|
+
exports.extractTopics = extractTopics;
|
|
6
|
+
exports.generatePersonas = generatePersonas;
|
|
5
7
|
exports.extractAndAssignTopics = extractAndAssignTopics;
|
|
6
8
|
exports.assignFunnelStages = assignFunnelStages;
|
|
7
9
|
exports.classifyIntoPersonas = classifyIntoPersonas;
|
|
@@ -16,20 +18,20 @@ const index_js_1 = require("./apis/chatgptScraper/index.js");
|
|
|
16
18
|
const classifier_js_1 = require("./tools/classifier.js");
|
|
17
19
|
const entities_js_1 = require("./tools/entities.js");
|
|
18
20
|
const funnel_js_1 = require("./tools/funnel.js");
|
|
19
|
-
const
|
|
21
|
+
const utils_js_1 = require("./helpers/utils.js");
|
|
20
22
|
const scorer_js_1 = require("./tools/scorer.js");
|
|
21
23
|
const topics_js_1 = require("./tools/topics.js");
|
|
22
|
-
const
|
|
24
|
+
const personas_js_1 = require("./tools/personas.js");
|
|
25
|
+
const utils_js_2 = require("./helpers/utils.js");
|
|
23
26
|
async function queryModel(prompts, model, searchCountry = null) {
|
|
24
|
-
const modelId = model instanceof models_js_1.ModelId ? model : new models_js_1.ModelId(model);
|
|
25
27
|
let searchResults;
|
|
26
|
-
if (
|
|
28
|
+
if (model === 'google/ai-overview') {
|
|
27
29
|
searchResults = await (0, aio_js_1.fetchAIOBatch)(prompts, searchCountry, null);
|
|
28
30
|
}
|
|
29
|
-
else if (
|
|
31
|
+
else if (model === 'google/ai-mode') {
|
|
30
32
|
searchResults = await (0, aim_js_1.fetchAIMBatch)(prompts, searchCountry, null);
|
|
31
33
|
}
|
|
32
|
-
else if (
|
|
34
|
+
else if (model.startsWith('openai/') || model.includes('chatgpt')) {
|
|
33
35
|
// JobIds are alphanumeric strings without spaces
|
|
34
36
|
// (e.g., "7420410504197219329" for Oxylabs, "s_xxxxx" for Brightdata)
|
|
35
37
|
// Prompts are natural language text with spaces
|
|
@@ -46,13 +48,15 @@ async function queryModel(prompts, model, searchCountry = null) {
|
|
|
46
48
|
}
|
|
47
49
|
}
|
|
48
50
|
else {
|
|
49
|
-
throw new Error(`Unsupported model: ${
|
|
51
|
+
throw new Error(`Unsupported model: ${model}`);
|
|
50
52
|
}
|
|
53
|
+
// Convert model identifier to column name (e.g., "google/ai-overview" -> "ai_overview")
|
|
54
|
+
const modelColumnName = (0, utils_js_1.cleanColumnName)(model.split('/').pop() ?? model);
|
|
51
55
|
const responses = searchResults.map((sr, i) => ({
|
|
52
56
|
answer: sr.answer,
|
|
53
57
|
sources: sr.sources,
|
|
54
58
|
prompt: sr.prompt ?? (typeof prompts[i] === 'string' ? prompts[i] : ''),
|
|
55
|
-
model:
|
|
59
|
+
model: modelColumnName,
|
|
56
60
|
searchQueries: sr.searchQueries ?? [],
|
|
57
61
|
searchSources: sr.searchSources ?? []
|
|
58
62
|
}));
|
|
@@ -65,20 +69,51 @@ const intentLabels = {
|
|
|
65
69
|
};
|
|
66
70
|
async function classifyIntent(texts) {
|
|
67
71
|
console.log('Classifying prompt intents...');
|
|
72
|
+
const classifier = new classifier_js_1.Classifier({
|
|
73
|
+
labels: intentLabels,
|
|
74
|
+
instructions: 'Classify the search query into one of the following intents: informational, navigational, transactional.'
|
|
75
|
+
}, { model: 'gpt-4.1-mini' });
|
|
68
76
|
const textRecords = texts.map(text => ({ text }));
|
|
69
|
-
const intents = await
|
|
70
|
-
return intents;
|
|
77
|
+
const intents = await classifier.batch(textRecords);
|
|
78
|
+
return intents.toArray();
|
|
79
|
+
}
|
|
80
|
+
async function extractTopics(params) {
|
|
81
|
+
const extractor = new topics_js_1.TopicExtractor({
|
|
82
|
+
maxSamples: params.maxSamples,
|
|
83
|
+
instructions: params.instructions,
|
|
84
|
+
language: params.language
|
|
85
|
+
}, { model: params.model ?? 'gpt-4.1-mini' });
|
|
86
|
+
const result = await extractor.invoke(params.records);
|
|
87
|
+
return result.parsed;
|
|
88
|
+
}
|
|
89
|
+
async function generatePersonas(params) {
|
|
90
|
+
const generator = new personas_js_1.PersonaGenerator({
|
|
91
|
+
sector: params.sector,
|
|
92
|
+
market: params.market,
|
|
93
|
+
language: params.language,
|
|
94
|
+
brand: params.brand,
|
|
95
|
+
brandDomain: params.brandDomain,
|
|
96
|
+
count: params.count,
|
|
97
|
+
briefing: params.briefing,
|
|
98
|
+
instructions: params.instructions,
|
|
99
|
+
userLanguage: params.userLanguage
|
|
100
|
+
}, { model: params.model ?? 'gpt-4.1' });
|
|
101
|
+
const result = await generator.invoke(params.personas ?? null);
|
|
102
|
+
return result.parsed;
|
|
71
103
|
}
|
|
72
104
|
async function extractAndAssignTopics(texts) {
|
|
73
105
|
console.log('Identifying topics...');
|
|
74
106
|
const textRecords = texts.map(text => ({ text }));
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
107
|
+
const extractor = new topics_js_1.TopicExtractor({ maxSamples: 500 }, { model: 'gpt-4.1' });
|
|
108
|
+
const taxonomyResult = await extractor.invoke(textRecords);
|
|
109
|
+
const taxonomy = taxonomyResult.parsed;
|
|
110
|
+
if (taxonomy == null) {
|
|
111
|
+
return texts.map(() => ({ topic: null, subtopic: null }));
|
|
112
|
+
}
|
|
79
113
|
console.log('Assigning topics...');
|
|
80
|
-
const
|
|
81
|
-
|
|
114
|
+
const assigner = new topics_js_1.TopicAssigner({ taxonomy }, { model: 'gpt-4.1-mini' });
|
|
115
|
+
const topicLabels = await assigner.batch(texts);
|
|
116
|
+
return topicLabels.toArray().map(label => ({
|
|
82
117
|
topic: label?.topic ?? null,
|
|
83
118
|
subtopic: label?.subtopic ?? null
|
|
84
119
|
}));
|
|
@@ -86,8 +121,9 @@ async function extractAndAssignTopics(texts) {
|
|
|
86
121
|
async function assignFunnelStages(texts, funnel) {
|
|
87
122
|
console.log('Assigning funnel stages...');
|
|
88
123
|
const funnelTopics = (0, funnel_js_1.funnelToTopics)(funnel);
|
|
89
|
-
const
|
|
90
|
-
|
|
124
|
+
const assigner = new topics_js_1.TopicAssigner({ taxonomy: funnelTopics }, { model: 'gpt-4.1-mini' });
|
|
125
|
+
const funnelLabels = await assigner.batch(texts);
|
|
126
|
+
return funnelLabels.toArray().map(label => ({
|
|
91
127
|
funnelStage: label?.topic ?? null,
|
|
92
128
|
funnelCategory: label?.subtopic ?? null
|
|
93
129
|
}));
|
|
@@ -98,9 +134,13 @@ async function classifyIntoPersonas(texts, personas) {
|
|
|
98
134
|
personas.forEach(persona => {
|
|
99
135
|
personaLabels[persona.name] = persona.description;
|
|
100
136
|
});
|
|
137
|
+
const labeler = new classifier_js_1.Labeler({
|
|
138
|
+
labels: personaLabels,
|
|
139
|
+
instructions: 'Classify the search query into one or more customer personas based on the language, intent, and context.'
|
|
140
|
+
}, { model: 'gpt-4.1-mini' });
|
|
101
141
|
const textRecords = texts.map(text => ({ text }));
|
|
102
|
-
const personaAssignments = await
|
|
103
|
-
return personaAssignments;
|
|
142
|
+
const personaAssignments = await labeler.batch(textRecords);
|
|
143
|
+
return personaAssignments.toArray();
|
|
104
144
|
}
|
|
105
145
|
async function classifyBrandedNonBranded(texts) {
|
|
106
146
|
console.log('Classifying prompts into branded/non-branded...');
|
|
@@ -108,29 +148,37 @@ async function classifyBrandedNonBranded(texts) {
|
|
|
108
148
|
'branded': 'The query explicitly mentions a specific brand name, company name, product name, or trademark. It shows clear brand awareness and intent to find information about that particular brand.',
|
|
109
149
|
'non-branded': 'The query is generic and does not mention any specific brand names. It focuses on product categories, features, problems, or general information without brand specificity.'
|
|
110
150
|
};
|
|
151
|
+
const classifier = new classifier_js_1.Classifier({
|
|
152
|
+
labels: brandedLabels,
|
|
153
|
+
instructions: 'Classify whether the search query mentions specific brands or is generic/category-based.'
|
|
154
|
+
}, { model: 'gpt-4.1-mini' });
|
|
111
155
|
const textRecords = texts.map(text => ({ text }));
|
|
112
|
-
const brandedClassifications = await
|
|
113
|
-
return brandedClassifications;
|
|
156
|
+
const brandedClassifications = await classifier.batch(textRecords);
|
|
157
|
+
return brandedClassifications.toArray();
|
|
114
158
|
}
|
|
115
159
|
async function extractEntities(texts) {
|
|
116
|
-
const
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
160
|
+
const extractor = new entities_js_1.EntityExtractor({
|
|
161
|
+
entityDefinitions: {
|
|
162
|
+
brands: 'Any brand or companies mentioned',
|
|
163
|
+
products: 'Any products or services mentioned',
|
|
164
|
+
features: 'Specific features or attributes of products/services',
|
|
165
|
+
issues: 'Problems or issues mentioned'
|
|
166
|
+
}
|
|
167
|
+
}, { model: 'gpt-4.1-mini' });
|
|
168
|
+
const result = await extractor.batch(texts);
|
|
169
|
+
return result.toArray();
|
|
123
170
|
}
|
|
124
171
|
async function scorePurchaseProbability(texts, numDays = 30) {
|
|
125
172
|
console.log('Scoring purchase probability...');
|
|
126
173
|
const records = texts.map(text => ({ text }));
|
|
127
|
-
const description = (0,
|
|
174
|
+
const description = (0, utils_js_2.dedent)(`
|
|
128
175
|
A score from 0 to 100 indicating the likelihood that the user intends to make a purchase
|
|
129
176
|
in the next ${numDays} days. The input record is a text query representing a search
|
|
130
177
|
that a user may perform using a traditional search engine or LLM chat.
|
|
131
178
|
`);
|
|
132
|
-
const
|
|
133
|
-
|
|
179
|
+
const scorer = new scorer_js_1.Scorer({ name: 'Purchase Probability', description: description, type: 'integer', min: 0, max: 100 }, { model: 'gpt-4.1-mini' });
|
|
180
|
+
const scores = await scorer.batch(records);
|
|
181
|
+
return scores.toArray();
|
|
134
182
|
}
|
|
135
183
|
async function scoreRelevance(prompts, context) {
|
|
136
184
|
console.log('Scoring prompt relevance...');
|
|
@@ -151,7 +199,7 @@ async function scoreRelevance(prompts, context) {
|
|
|
151
199
|
? `Analysis context:\n${contextParts.join('\n')}`
|
|
152
200
|
: '';
|
|
153
201
|
const records = prompts.map(prompt => ({ prompt }));
|
|
154
|
-
const description = (0,
|
|
202
|
+
const description = (0, utils_js_2.dedent)(`
|
|
155
203
|
A score from 0.0 to 1.0 measuring how relevant this search keyword/prompt is
|
|
156
204
|
for the specified SEO analysis context. Consider whether it is specific enough
|
|
157
205
|
for the following brand, sector, market and research brief.
|
|
@@ -167,6 +215,7 @@ async function scoreRelevance(prompts, context) {
|
|
|
167
215
|
1.0 means the keyword/prompt is highly relevant and specific to the context, e.g.
|
|
168
216
|
"best CRM software for small business" for a brand selling CRM software.
|
|
169
217
|
`);
|
|
170
|
-
const
|
|
171
|
-
|
|
218
|
+
const scorer = new scorer_js_1.Scorer({ name: 'Prompt Relevance', description: description, type: 'number', min: 0.0, max: 1.0 }, { model: 'gpt-4.1-mini' });
|
|
219
|
+
const scores = await scorer.batch(records);
|
|
220
|
+
return scores.toArray();
|
|
172
221
|
}
|
|
@@ -44,7 +44,7 @@ exports.brightdataProvider = void 0;
|
|
|
44
44
|
* 3. Download: GET /datasets/v3/snapshot/{snapshot_id}
|
|
45
45
|
*/
|
|
46
46
|
const dntShim = __importStar(require("../../../_dnt.shims.js"));
|
|
47
|
-
const async_js_1 = require("../../async.js");
|
|
47
|
+
const async_js_1 = require("../../helpers/async.js");
|
|
48
48
|
const scraper_js_1 = require("./scraper.js");
|
|
49
49
|
// ============================================================================
|
|
50
50
|
// Constants
|
|
@@ -44,7 +44,7 @@ exports.oxylabsProvider = void 0;
|
|
|
44
44
|
* 3. Download: GET /v1/queries/{id}/results
|
|
45
45
|
*/
|
|
46
46
|
const dntShim = __importStar(require("../../../_dnt.shims.js"));
|
|
47
|
-
const async_js_1 = require("../../async.js");
|
|
47
|
+
const async_js_1 = require("../../helpers/async.js");
|
|
48
48
|
const scraper_js_1 = require("./scraper.js");
|
|
49
49
|
// ============================================================================
|
|
50
50
|
// Constants
|
|
@@ -46,8 +46,8 @@ exports.createScraper = createScraper;
|
|
|
46
46
|
* Uses composition: providers supply functions, this module orchestrates them.
|
|
47
47
|
*/
|
|
48
48
|
const dntShim = __importStar(require("../../../_dnt.shims.js"));
|
|
49
|
-
const async_js_1 = require("../../async.js");
|
|
50
|
-
const urls_js_1 = require("../../urls.js");
|
|
49
|
+
const async_js_1 = require("../../helpers/async.js");
|
|
50
|
+
const urls_js_1 = require("../../helpers/urls.js");
|
|
51
51
|
// ============================================================================
|
|
52
52
|
// Shared Utilities
|
|
53
53
|
// ============================================================================
|
|
@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.fetchAIM = fetchAIM;
|
|
4
4
|
exports.fetchAIMBatch = fetchAIMBatch;
|
|
5
5
|
/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
|
|
6
|
-
const async_js_1 = require("../../async.js");
|
|
6
|
+
const async_js_1 = require("../../helpers/async.js");
|
|
7
7
|
const helpers_js_1 = require("./helpers.js");
|
|
8
8
|
async function fetchAIM(prompt, country = null, language = null, location = null) {
|
|
9
9
|
const aimEndpoint = 'https://api.hasdata.com/scrape/google/ai-mode';
|
|
@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.fetchAIO = fetchAIO;
|
|
4
4
|
exports.fetchAIOBatch = fetchAIOBatch;
|
|
5
5
|
/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
|
|
6
|
-
const async_js_1 = require("../../async.js");
|
|
6
|
+
const async_js_1 = require("../../helpers/async.js");
|
|
7
7
|
const helpers_js_1 = require("./helpers.js");
|
|
8
8
|
function aioRequestUrl(aio) {
|
|
9
9
|
if (aio.pageToken && aio.hasdataLink) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type RetryConfig } from '../../async.js';
|
|
1
|
+
import { type RetryConfig } from '../../helpers/async.js';
|
|
2
2
|
import type { Source } from '../../schemas/sources.schema.js';
|
|
3
3
|
export declare const HASDATA_CONCURRENCY = 29;
|
|
4
4
|
export declare const HASDATA_RETRY_CONFIG: RetryConfig;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;AAwJD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}
|
|
@@ -39,8 +39,8 @@ exports.fetchHasDataWithRetry = fetchHasDataWithRetry;
|
|
|
39
39
|
exports.parseAIO = parseAIO;
|
|
40
40
|
exports.parseAIM = parseAIM;
|
|
41
41
|
const dntShim = __importStar(require("../../../_dnt.shims.js"));
|
|
42
|
-
const async_js_1 = require("../../async.js");
|
|
43
|
-
const urls_js_1 = require("../../urls.js");
|
|
42
|
+
const async_js_1 = require("../../helpers/async.js");
|
|
43
|
+
const urls_js_1 = require("../../helpers/urls.js");
|
|
44
44
|
exports.HASDATA_CONCURRENCY = 29;
|
|
45
45
|
exports.HASDATA_RETRY_CONFIG = {
|
|
46
46
|
maxRetries: 3,
|