@graphext/cuery 0.2.0

package/esm/src/apis/chatgptScraper/scraper.js

@@ -0,0 +1,98 @@
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+/**
+ * GPT Scraper - Core types and orchestration logic.
+ *
+ * Uses composition: providers supply functions, this module orchestrates them.
+ */
+import * as dntShim from "../../../_dnt.shims.js";
+import { mapParallel } from '../../async.js';
+import { extractDomain } from '../../urls.js';
+// ============================================================================
+// Shared Utilities
+// ============================================================================
+export function getAbortSignal() {
+    return dntShim.dntGlobalThis.abortSignal;
+}
+export function cleanAnswer(answer) {
+    return answer
+        .replace(/!\[([^\]]*)\]\([^)]+\)/g, '')
+        .replace(/\n\s*Image\s*\n/g, '\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}
+export function buildSources(citations, linkPositions) {
+    return citations.map(c => ({
+        title: c.title || c.description || c.text || '',
+        url: c.url,
+        domain: extractDomain(c.url),
+        cited: c.cited,
+        positions: linkPositions?.[c.url]
+    }));
+}
+export function buildSearchSources(sources) {
+    return sources.map(s => ({
+        title: s.title || s.snippet || '',
+        url: s.url || '',
+        domain: s.url ? extractDomain(s.url) : '',
+        rank: s.rank || 0,
+        datePublished: s.date_published || null
+    }));
+}
+/**
+ * Creates an empty model result for failed jobs.
+ * This ensures we always return the same number of rows as input.
+ */
+export function emptyModelResult(providerName, errorMessage, context) {
+    if (errorMessage) {
+        console.error(`[${providerName}] ${errorMessage}`, context ?? '');
+    }
+    return {
+        prompt: '',
+        answer: '',
+        sources: []
+    };
+}
+// ============================================================================
+// Scraper Factory
+// ============================================================================
+export function createScraper(provider) {
+    const { name, maxConcurrency, maxPromptsPerRequest, triggerJob, monitorJob, downloadJob, transformResponse } = provider;
+    async function triggerGPTBatch({ prompts, useSearch = false, countryISOCode = null }) {
+        const jobIds = await mapParallel(prompts, maxConcurrency, (prompt) => triggerJob(prompt, useSearch, countryISOCode));
+        console.log(`[${name}] Triggered ${jobIds.length} jobs for ${prompts.length} prompts`);
+        return jobIds;
+    }
+    async function downloadGPTSnapshots(jobIds) {
+        const results = [];
+        for (const jobId of jobIds) {
+            if (!jobId) {
+                results.push(emptyModelResult(name, 'No job ID provided'));
+                continue;
+            }
+            const isReady = await monitorJob(jobId);
+            if (!isReady) {
+                results.push(emptyModelResult(name, 'Job not ready or failed', jobId));
+                continue;
+            }
+            const raw = await downloadJob(jobId);
+            if (!raw) {
+                results.push(emptyModelResult(name, 'Failed to download job', jobId));
+                continue;
+            }
+            const result = transformResponse(raw);
+            results.push(result ?? emptyModelResult(name, 'Failed to transform response', jobId));
+        }
+        return results;
+    }
+    async function scrapeGPTBatch(options) {
+        const jobIds = await triggerGPTBatch(options);
+        return downloadGPTSnapshots(jobIds);
+    }
+    return {
+        maxConcurrency,
+        maxPromptsPerRequest,
+        scrapeGPTBatch,
+        triggerGPTBatch,
+        downloadGPTSnapshots
+    };
+}

package/esm/src/apis/googleAds/apiTypes.d.ts

@@ -0,0 +1,131 @@
+/**
+ * @link https://developers.google.com/google-ads/api/reference/rpc/v21/KeywordPlanNetworkEnum.KeywordPlanNetwork
+ */
+export type KeywordPlanNetwork = 'UNSPECIFIED' | 'UNKNOWN' | 'GOOGLE_SEARCH' | 'GOOGLE_SEARCH_AND_PARTNERS';
+/**
+ * @link https://developers.google.com/google-ads/api/reference/rpc/v21/KeywordPlanKeywordAnnotationEnum.KeywordPlanKeywordAnnotation
+ */
+export type KeywordPlanKeywordAnnotation = 'UNSPECIFIED' | 'UNKNOWN' | 'KEYWORD_CONCEPT';
+/**
+ * @link https://developers.google.com/google-ads/api/reference/rpc/v21/KeywordPlanCompetitionLevelEnum.KeywordPlanCompetitionLevel
+ */
+export type KeywordPlanCompetitionLevel = 'UNSPECIFIED' | 'UNKNOWN' | 'LOW' | 'MEDIUM' | 'HIGH';
+/**
+ * @link https://developers.google.com/google-ads/api/reference/rpc/v21/KeywordPlanIdeaService/GenerateKeywordIdeas?transport=rest#keywordplanconceptgrouptype
+ */
+type KeywordPlanConceptGroupType = 'UNSPECIFIED' | 'UNKNOWN' | 'BRAND' | 'OTHER_BRANDS' | 'NON_BRAND';
+/**
+ * @link https://developers.google.com/google-ads/api/reference/rpc/v21/MonthOfYearEnum.MonthOfYear
+ */
+export type MonthOfYear = 'UNSPECIFIED' | 'UNKNOWN' | 'JANUARY' | 'FEBRUARY' | 'MARCH' | 'APRIL' | 'MAY' | 'JUNE' | 'JULY' | 'AUGUST' | 'SEPTEMBER' | 'OCTOBER' | 'NOVEMBER' | 'DECEMBER';
+/**
+ * @link https://developers.google.com/google-ads/api/reference/rpc/v21/KeywordPlanIdeaService/GenerateKeywordIdeas?transport=rest#request-body
+ */
+export interface GenerateKeywordIdeasRequest {
+    language?: string;
+    geoTargetConstants?: Array<string>;
+    includeAdultKeywords?: boolean;
+    pageToken?: string;
+    pageSize?: number;
+    keywordPlanNetwork?: KeywordPlanNetwork;
+    keywordAnnotation?: Array<KeywordPlanKeywordAnnotation>;
+    aggregateMetrics?: {
+        aggregate_metric_types: Array<'UNSPECIFIED' | 'UNKNOWN' | 'DEVICE'>;
+    };
+    historicalMetricsOptions?: {
+        include_average_cpc?: boolean;
+        year_month_range?: {
+            start?: {
+                year: number;
+                month: MonthOfYear;
+            };
+            end?: {
+                year: number;
+                month: MonthOfYear;
+            };
+        };
+    };
+    keywordAndUrlSeed?: {
+        url?: string;
+        keywords?: Array<string>;
+    };
+    keywordSeed?: {
+        keywords: Array<string>;
+    };
+    urlSeed?: {
+        url: string;
+    };
+    siteSeed?: {
+        site: string;
+    };
+}
+/**
+ * @link https://developers.google.com/google-ads/api/reference/rpc/v21/KeywordPlanIdeaService/GenerateKeywordHistoricalMetrics?transport=rest#request-body
+ */
+export interface GenerateKeywordHistoricalMetricsRequest {
+    keywords?: Array<string>;
+    language?: string;
+    includeAdultKeywords?: boolean;
+    geoTargetConstants?: Array<string>;
+    keywordPlanNetwork?: KeywordPlanNetwork;
+    aggregateMetrics?: {
+        aggregate_metric_types: Array<'UNSPECIFIED' | 'UNKNOWN' | 'DEVICE'>;
+    };
+    historicalMetricsOptions?: {
+        include_average_cpc?: boolean;
+        year_month_range?: {
+            start?: {
+                year: number;
+                month: MonthOfYear;
+            };
+            end?: {
+                year: number;
+                month: MonthOfYear;
+            };
+        };
+    };
+}
+export interface KeywordIdeaResult {
+    text?: string | null;
+    keywordIdeaMetrics?: KeywordMetrics | null;
+    keywordAnnotations?: KeywordAnnotations | null;
+    closeVariants?: Array<string> | null;
+}
+export interface HistoricalMetricsResult {
+    text?: string | null;
+    closeVariants?: Array<string> | null;
+    keywordMetrics?: KeywordMetrics | null;
+}
+export type KeywordResult = KeywordIdeaResult | HistoricalMetricsResult;
+export interface GenerateKeywordHistoricalMetricsResponse {
+    results: Array<HistoricalMetricsResult>;
+}
+export interface GenerateKeywordIdeaResponse {
+    results: Array<KeywordIdeaResult>;
+}
+export type KeywordServiceResponse = GenerateKeywordHistoricalMetricsResponse | GenerateKeywordIdeaResponse;
+export interface KeywordAnnotations {
+    concepts?: Array<{
+        name?: string | null;
+        conceptGroup?: {
+            name?: string | null;
+            type?: KeywordPlanConceptGroupType | null;
+        } | null;
+    }> | null;
+}
+export interface MonthlySearchVolume {
+    year?: number | null;
+    month?: MonthOfYear | null;
+    monthlySearches?: number | null;
+}
+export interface KeywordMetrics {
+    avgMonthlySearches?: number | null;
+    monthlySearchVolumes?: Array<MonthlySearchVolume> | null;
+    competition?: KeywordPlanCompetitionLevel | null;
+    competitionIndex?: number | null;
+    lowTopOfPageBidMicros?: number | null;
+    highTopOfPageBidMicros?: number | null;
+    averageCpcMicros?: number | null;
+}
+export {};
+//# sourceMappingURL=apiTypes.d.ts.map

package/esm/src/apis/googleAds/apiTypes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"apiTypes.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/googleAds/apiTypes.ts"],"names":[],"mappings":"AACA;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAC3B,aAAa,GACb,SAAS,GACT,eAAe,GACf,4BAA4B,CAAC;AAEhC;;GAEG;AACH,MAAM,MAAM,4BAA4B,GACrC,aAAa,GACb,SAAS,GACT,iBAAiB,CAAC;AAErB;;GAEG;AACH,MAAM,MAAM,2BAA2B,GACpC,aAAa,GACb,SAAS,GACT,KAAK,GACL,QAAQ,GACR,MAAM,CAAC;AAEV;;GAEG;AACH,KAAK,2BAA2B,GAC7B,aAAa,GACb,SAAS,GACT,OAAO,GACP,cAAc,GACd,WAAW,CAAC;AAEf;;GAEG;AACH,MAAM,MAAM,WAAW,GACpB,aAAa,GACb,SAAS,GACT,SAAS,GACT,UAAU,GACV,OAAO,GACP,OAAO,GACP,KAAK,GACL,MAAM,GACN,MAAM,GACN,QAAQ,GACR,WAAW,GACX,SAAS,GACT,UAAU,GACV,UAAU,CAAC;AAEd;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC3C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACnC,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IACxC,iBAAiB,CAAC,EAAE,KAAK,CAAC,4BAA4B,CAAC,CAAC;IACxD,gBAAgB,CAAC,EAAE;QAClB,sBAAsB,EAAE,KAAK,CAAC,aAAa,GAAG,SAAS,GAAG,QAAQ,CAAC,CAAA;KACnE,CAAC;IACF,wBAAwB,CAAC,EAAE;QAC1B,mBAAmB,CAAC,EAAE,OAAO,CAAC;QAC9B,gBAAgB,CAAC,EAAE;YAClB,KAAK,CAAC,EAAE;gBACP,IAAI,EAAE,MAAM,CAAC;gBACb,KAAK,EAAE,WAAW,CAAC;aACnB,CAAC;YACF,GAAG,CAAC,EAAE;gBACL,IAAI,EAAE,MAAM,CAAC;gBACb,KAAK,EAAE,WAAW,CAAC;aACnB,CAAC;SACF,CAAC;KACF,CAAC;IACF,iBAAiB,CAAC,EAAE;QACnB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,QAAQ,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;KACzB,CAAC;IACF,WAAW,CAAC,EAAE;QACb,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;KACxB,CAAC;IACF,OAAO,CAAC,EAAE;QACR,GAAG,EAAE,MAAM,CAAC;KACb,CAAC;IACF,QAAQ,CAAC,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;KACb,CAAC;CACF;AAED;;GAEG;AACH,MAAM,WAAW,uCAAuC;IACvD,QAAQ,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,kBAAkB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACnC,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IACxC,gBAAgB,CAAC,EAAE;QAClB,sBAAsB,EAAE,KAAK,CAAC,aAAa,GAAG,SAAS,GAAG,QAAQ,CAAC,CAAC;KACpE,CAAC;IACF,wBAAwB,CAAC,EAAE;QAC1B,mBAAmB,CAAC,EAAE,OAAO,CAAC;QAC9B,gBAAgB,CAAC,EAAE;YAClB,KAAK,CAAC,EAAE;gBACP,IAAI,EAAE,MAAM,CAAC;gBACb,KAAK,EAAE,WAAW,CAAC;aACnB,CAAC;YACF,GAAG,CAAC,EAAE;gBACL,IAAI,EAAE,MAAM,CAAC;gBACb,KAAK,EAAE,WAAW,CAAC;aACnB,CAAC;SACF,CAAC;KACF,CAAC;CACF;AAED,MAAM,WAAW,iBAAiB;IACjC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,kBAAkB,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;IAC3C,kBAAkB,CAAC,EAAE,kBAAkB,GAAG,IAAI,CAAC;IAC/C,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;CACrC;AAED,MAAM,WAAW,uBAAuB;IACvC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;IACrC,cAAc,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;CACvC;AAED,MAAM,MAAM,aAAa,GAAG,iBAAiB,GAAG,uBAAuB,CAAC;AAExE,MAAM,WAAW,wCAAwC;IACxD,OAAO,EAAE,KAAK,CAAC,uBAAuB,CAAC,CAAC;CACxC;AAED,MAAM,WAAW,2BAA2B;IAC3C,OAAO,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC;CAClC;AAED,MAAM,MAAM,sBAAsB,GAC/B,wCAAwC,GACxC,2BAA2B,CAAC;AAE/B,MAAM,WAAW,kBAAkB;IAClC,QAAQ,CAAC,EAAE,KAAK,CAAC;QAChB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QACrB,YAAY,CAAC,EAAE;YACd,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;YACrB,IAAI,CAAC,EAAE,2BAA2B,GAAG,IAAI,CAAC;SAC1C,GAAG,IAAI,CAAC;KACT,CAAC,GAAG,IAAI,CAAC;CACV;AAED,MAAM,WAAW,mBAAmB;IACnC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,KAAK,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED,MAAM,WAAW,cAAc;IAC9B,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,oBAAoB,CAAC,EAAE,KAAK,CAAC,mBAAmB,CAAC,GAAG,IAAI,CAAC;IACzD,WAAW,CAAC,EAAE,2BAA2B,GAAG,IAAI,CAAC;IACjD,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,qBAAqB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,sBAAsB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC"}

package/esm/src/apis/googleAds/apiTypes.js

@@ -0,0 +1 @@
+export {};

package/esm/src/apis/googleAds/client.d.ts

@@ -0,0 +1,2 @@
+export declare function createGoogleAdsClient(): Promise<((apiMethod: string, params: Record<string, unknown>) => Promise<unknown>) | ((apiMethod: string, params: Record<string, any>) => Promise<any>)>;
+//# sourceMappingURL=client.d.ts.map

package/esm/src/apis/googleAds/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/googleAds/client.ts"],"names":[],"mappings":"AA4EA,wBAAsB,qBAAqB,yBAJtB,MAAM,UAAU,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,iBAqB/C,MAAM,UAAU,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,oBAyCpE"}

package/esm/src/apis/googleAds/client.js

@@ -0,0 +1,97 @@
+// Required env vars:
+// GOOGLE_ADS_DEVELOPER_TOKEN,
+// GOOGLE_ADS_SERVICE_ACCOUNT_JSON_BASE64,
+// GOOGLE_ADS_LOGIN_CUSTOMER_ID or GOOGLE_ADS_CUSTOMER_ID
+//
+// Optional env vars:
+// GOOGLE_ADS_LOGIN_CUSTOMER_ID (MCC manager account signing requests)
+// GOOGLE_ADS_LINKED_CUSTOMER_ID (execute within linked account context).
+import * as dntShim from "../../../_dnt.shims.js";
+import { JWT } from 'google-auth-library';
+const MAX_RETRIES = 50;
+function normalizeCustomerId(id) {
+    return id.replace(/-/g, '');
+}
+function assertEnv(name) {
+    const value = dntShim.Deno.env.get(name);
+    if (!value) {
+        throw new Error(`Environment variable ${name} is required to access the Google Ads API.`);
+    }
+    return value;
+}
+function getCredentialsFromEnv() {
+    const developerToken = assertEnv('GOOGLE_ADS_DEVELOPER_TOKEN');
+    // For Service Account authentication:
+    const serviceAccountJson = JSON.parse(atob(assertEnv('GOOGLE_ADS_SERVICE_ACCOUNT_JSON_BASE64')));
+    // For OAuth2 authentication:
+    // const clientId = assertEnv('GOOGLE_ADS_CLIENT_ID');
+    // const clientSecret = assertEnv('GOOGLE_ADS_CLIENT_SECRET');
+    // const refreshToken = assertEnv('GOOGLE_ADS_REFRESH_TOKEN');
+    const loginCustomerId = dntShim.Deno.env.get('GOOGLE_ADS_LOGIN_CUSTOMER_ID');
+    const linkedCustomerId = dntShim.Deno.env.get('GOOGLE_ADS_LINKED_CUSTOMER_ID');
+    const rawCustomerId = dntShim.Deno.env.get('GOOGLE_ADS_CUSTOMER_ID') ?? loginCustomerId ?? '';
+    const customerId = rawCustomerId.trim();
+    if (!customerId) {
+        throw new Error('Environment variable GOOGLE_ADS_CUSTOMER_ID is required to access the Google Ads API.');
+    }
+    return {
+        developerToken,
+        serviceAccountJson,
+        customerId: normalizeCustomerId(customerId),
+        loginCustomerId: loginCustomerId ? normalizeCustomerId(loginCustomerId) : undefined,
+        linkedCustomerId: linkedCustomerId ? normalizeCustomerId(linkedCustomerId) : undefined
+    };
+}
+let cachedClient = null;
+export async function createGoogleAdsClient() {
+    if (cachedClient
+        && cachedClient.expiration
+        && cachedClient.expiration > Date.now() + 60000) {
+        return cachedClient.client;
+    }
+    const credentials = getCredentialsFromEnv();
+    const authClient = new JWT({
+        email: credentials.serviceAccountJson.client_email,
+        key: credentials.serviceAccountJson.private_key,
+        scopes: ['https://www.googleapis.com/auth/adwords']
+    });
+    const { access_token, expiry_date: expiration } = await authClient.authorize();
+    const client = async (apiMethod, params) => {
+        let retries = MAX_RETRIES;
+        while (retries > 0) {
+            const response = await fetch(`https://googleads.googleapis.com/v21/customers/${credentials.customerId}:${apiMethod}`, {
+                method: 'POST',
+                headers: {
+                    'Authorization': `Bearer ${access_token}`,
+                    'Content-Type': 'application/json',
+                    'developer-token': credentials.developerToken,
+                    'customer_id': credentials.customerId,
+                    'login-customer-id': credentials.loginCustomerId ?? ''
+                },
+                body: JSON.stringify(params),
+                signal: dntShim.dntGlobalThis.abortSignal // Pass through abort signal if available
+            });
+            if (!response.ok) {
+                if (response.status === 429) {
+                    // Rate limit exceeded, retry after a delay
+                    const delay = ((MAX_RETRIES + 1) - retries) * 500; // Exponential backoff
+                    console.log('Rate limit exceeded, retrying after', delay, 'ms');
+                    await new Promise(res => setTimeout(res, delay));
+                    retries--;
+                    continue;
+                }
+                throw new Error(`Google Ads API error: ${response.statusText} (response code: ${response.status})`);
+            }
+            try {
+                return response.json();
+            }
+            catch (error) {
+                throw new Error(`Failed to parse Google Ads API response: ${error}`);
+            }
+        }
+    };
+    if (expiration != null) {
+        cachedClient = { client, expiration };
+    }
+    return client;
+}

package/esm/src/apis/googleAds/countryResourceMap.d.ts

@@ -0,0 +1,2 @@
+export declare const COUNTRY_RESOURCE_MAP: Readonly<Record<string, string>>;
+//# sourceMappingURL=countryResourceMap.d.ts.map

package/esm/src/apis/googleAds/countryResourceMap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"countryResourceMap.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/googleAds/countryResourceMap.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,oBAAoB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAu2BhE,CAAC"}