google-ads-api 14.2.0 → 16.0.0-rest-beta0

package/README.md

@@ -7,7 +7,7 @@
 </p>
 <p align="center">
   <a href="https://developers.google.com/google-ads/api/docs/release-notes">
-    <img src="https://img.shields.io/badge/google%20ads-v14.1.0-009688.svg?style=flat-square">
+    <img src="https://img.shields.io/badge/google%20ads-v16.1.0-009688.svg?style=flat-square">
   </a>
   <a href="https://www.npmjs.com/package/google-ads-api">
     <img src="https://img.shields.io/npm/v/google-ads-api.svg?style=flat-square">
@@ -384,7 +384,7 @@ const adGroupResourceName = 'customers/123/adGroups/456'
 const keyword = '24 hour locksmith harlem'
 
 const operations: MutateOperation<
-  resources.IAdGroupCriterion & { exempt_policy_violation_keys?: google.ads.googleads.v14.common.IPolicyViolationKey[]}
+  resources.IAdGroupCriterion & { exempt_policy_violation_keys?: google.ads.googleads.v16.common.IPolicyViolationKey[]}
 >[] = [
   {
     entity: 'ad_group_criterion',

package/build/src/client.d.ts

@@ -7,6 +7,7 @@ export interface ClientOptions {
     client_secret: string;
     developer_token: string;
     disable_parsing?: boolean;
+    max_reporting_rows?: number;
 }
 export declare class Client {
     private readonly options;

package/build/src/customer.d.ts

@@ -4,6 +4,21 @@ import { Hooks } from "./hooks";
 import { services } from "./protos";
 import ServiceFactory from "./protos/autogen/serviceFactory";
 import { CustomerOptions, MutateOperation, MutateOptions, ReportOptions, RequestOptions } from "./types";
+/**
+ * TODO:
+ *  - rename querierRest to streamRest. Add hooks support to it. OK
+ *  - write querierRest, using pagination. OK
+ *  - make sure return_total_results_count works
+ *  - make sure SUMMARY_ROW_SETTING works
+ *  - adapt query()
+ *  - adapt queryStream
+ *  - adapt report()
+ *  - adapt reportCount()
+ *  - adapt reportStream()
+ *  - reportStreamRaw() ?
+ *  - bonus: use queryStream when the params allow it. Note: do it with native async generator to make it as fast as possible.
+ *  - bonus: make it possible to cancel a stream or pagination when row count exceeds specified max
+ */
 export declare class Customer extends ServiceFactory {
     constructor(clientOptions: ClientOptions, customerOptions: CustomerOptions, hooks?: Hooks);
     /**
@@ -52,8 +67,10 @@ export declare class Customer extends ServiceFactory {
     reportStreamRaw(reportOptions: Readonly<ReportOptions>): Promise<CancellableStream | void>;
     private search;
     private paginatedSearch;
+    private useStreamToImitateRegularSearch;
     private querier;
     private streamer;
+    private handleStreamError;
     /**
      * @description Creates, updates, or removes resources. This method supports atomic transactions
      * with multiple types of resources. For example, you can atomically create a campaign and a
@@ -62,4 +79,8 @@ export declare class Customer extends ServiceFactory {
      */
     mutateResources<T>(mutations: MutateOperation<T>[], mutateOptions?: MutateOptions): Promise<services.MutateGoogleAdsResponse>;
     private get googleAdsFields();
+    private prepareGoogleAdsServicePostRequestArgs;
+    private decamelizeKeysIfNeeded;
+    private gaqlQueryStringIncludesLimit;
+    private generateTooManyRowsError;
 }

package/build/src/customer.js

@@ -4,10 +4,31 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Customer = void 0;
-const parser_1 = require("./parser");
+const axios_1 = __importDefault(require("axios"));
+const stream_chain_1 = require("stream-chain");
+const stream_json_1 = require("stream-json");
+const StreamArray_1 = require("stream-json/streamers/StreamArray");
+const parserRest_1 = require("./parserRest");
+const protos_1 = require("./protos");
 const serviceFactory_1 = __importDefault(require("./protos/autogen/serviceFactory"));
 const query_1 = require("./query");
-const utils_1 = require("./utils");
+const version_1 = require("./version");
+const ROWS_PER_STREAMED_CHUNK = 10000; // From experience, this is what can be expected from the API.
+/**
+ * TODO:
+ *  - rename querierRest to streamRest. Add hooks support to it. OK
+ *  - write querierRest, using pagination. OK
+ *  - make sure return_total_results_count works
+ *  - make sure SUMMARY_ROW_SETTING works
+ *  - adapt query()
+ *  - adapt queryStream
+ *  - adapt report()
+ *  - adapt reportCount()
+ *  - adapt reportStream()
+ *  - reportStreamRaw() ?
+ *  - bonus: use queryStream when the params allow it. Note: do it with native async generator to make it as fast as possible.
+ *  - bonus: make it possible to cancel a stream or pagination when row count exceeds specified max
+ */
 class Customer extends serviceFactory_1.default {
     constructor(clientOptions, customerOptions, hooks) {
         super(clientOptions, customerOptions, hooks !== null && hooks !== void 0 ? hooks : {});
@@ -111,39 +132,119 @@ class Customer extends serviceFactory_1.default {
         });
     }
     async search(gaqlQuery, requestOptions) {
-        const { service, request } = this.buildSearchRequestAndService(gaqlQuery, requestOptions);
-        const searchResponse = await service.search(request, {
-            otherArgs: { headers: this.callHeaders },
-            autoPaginate: false, // autoPaginate doesn't work
-        });
-        const response = searchResponse[0];
-        const summaryRow = searchResponse[2].summary_row;
-        const nextPageToken = searchResponse[2].next_page_token;
-        const totalResultsCount = searchResponse[2].total_results_count
-            ? +searchResponse[2].total_results_count
-            : undefined;
-        if (summaryRow) {
-            response.unshift(summaryRow);
+        var _a, _b;
+        const accessToken = await this.getAccessToken();
+        try {
+            const rawResponse = await (0, axios_1.default)(this.prepareGoogleAdsServicePostRequestArgs("search", accessToken, {
+                data: {
+                    query: gaqlQuery,
+                    ...requestOptions,
+                },
+            }));
+            const searchResponse = rawResponse.data;
+            const results = (_a = searchResponse.results) !== null && _a !== void 0 ? _a : [];
+            const response = results.map((row) => this.decamelizeKeysIfNeeded(row));
+            const summaryRow = this.decamelizeKeysIfNeeded(searchResponse.summaryRow);
+            const nextPageToken = searchResponse.nextPageToken;
+            const totalResultsCount = searchResponse.totalResultsCount
+                ? +searchResponse.totalResultsCount
+                : undefined;
+            return { response, nextPageToken, totalResultsCount, summaryRow };
+        }
+        catch (e) {
+            if ((_b = e.response) === null || _b === void 0 ? void 0 : _b.data.error.details[0]) {
+                throw new protos_1.errors.GoogleAdsFailure(this.decamelizeKeysIfNeeded(e.response.data.error.details[0]));
+            }
+            throw e;
         }
-        return { response, nextPageToken, totalResultsCount };
     }
-    async paginatedSearch(gaqlQuery, requestOptions, parser) {
+    async paginatedSearch(gaqlQuery, requestOptions) {
+        /*
+          When possible, use the searchStream method to avoid the overhead of pagination.
+        */
+        if (requestOptions.page_size === undefined &&
+            // @ts-expect-error we do not allow this field in reportOptions, however it is still a valid request option
+            requestOptions.return_total_results_count === undefined) {
+            // If no pagination or summary options are set, we can use the non-paginated search method.
+            const { response } = await this.useStreamToImitateRegularSearch(gaqlQuery, requestOptions);
+            return { response };
+        }
         const response = [];
         let nextPageToken = undefined;
         const initialSearch = await this.search(gaqlQuery, requestOptions);
         const totalResultsCount = initialSearch.totalResultsCount;
-        response.push(...parser(initialSearch.response));
+        let summaryRow = initialSearch.summaryRow;
+        response.push(...initialSearch.response);
         nextPageToken = initialSearch.nextPageToken;
         while (nextPageToken) {
             const nextSearch = await this.search(gaqlQuery, {
                 ...requestOptions,
                 page_token: nextPageToken,
             });
-            response.push(...parser(nextSearch.response));
+            response.push(...nextSearch.response);
             nextPageToken = nextSearch.nextPageToken;
+            if (nextSearch.summaryRow) {
+                summaryRow = nextSearch.summaryRow;
+            }
+        }
+        if (summaryRow) {
+            response.unshift(summaryRow);
         }
         return { response, totalResultsCount };
     }
+    // Google's searchStream method is faster than search, but it does not support all features.
+    // When report() is called, we use searchStream if possible, otherwise we use paginatedSearch.
+    // Note that just like `paginatedSearch`, this method accumulates results in memory. Use
+    // `reportStream` for a more memory-efficient alternative (at the cost of more CPU usage).
+    async useStreamToImitateRegularSearch(gaqlQuery, requestOptions) {
+        const accessToken = await this.getAccessToken();
+        try {
+            const args = this.prepareGoogleAdsServicePostRequestArgs("searchStream", accessToken, {
+                responseType: "stream",
+                data: {
+                    query: gaqlQuery,
+                    ...requestOptions,
+                },
+            });
+            const response = await (0, axios_1.default)(args);
+            const stream = response.data;
+            const buffers = [];
+            let rowCount = -ROWS_PER_STREAMED_CHUNK;
+            for await (const data of stream) {
+                if (this.clientOptions.max_reporting_rows &&
+                    !this.gaqlQueryStringIncludesLimit(gaqlQuery)) {
+                    // This is a quick-and-dirty way to count rows, but it's good enough for our purposes.
+                    // We want to avoid using a proper JSON streamer here for performance reasons.
+                    if (data.toString("utf-8").includes(`results":`)) {
+                        rowCount += ROWS_PER_STREAMED_CHUNK;
+                    }
+                    if (rowCount > this.clientOptions.max_reporting_rows) {
+                        throw this.generateTooManyRowsError();
+                    }
+                }
+                buffers.push(data);
+            }
+            const asString = Buffer.concat(buffers).toString("utf-8");
+            const accumulator = [];
+            let foundSummaryRow;
+            for (const { results, summaryRow } of JSON.parse(asString)) {
+                if (summaryRow) {
+                    foundSummaryRow = this.decamelizeKeysIfNeeded(summaryRow);
+                }
+                accumulator.push(...(results !== null && results !== void 0 ? results : []).map((row) => {
+                    return this.decamelizeKeysIfNeeded(row);
+                }));
+                if (foundSummaryRow) {
+                    accumulator.unshift(foundSummaryRow);
+                }
+            }
+            return { response: accumulator };
+        }
+        catch (e) {
+            await this.handleStreamError(e);
+            throw e; // The line above should always throw.
+        }
+    }
     async querier(gaqlQuery, requestOptions = {}, reportOptions, useHooks = true) {
         const baseHookArguments = {
             credentials: this.credentials,
@@ -170,14 +271,7 @@ class Customer extends serviceFactory_1.default {
             }
         }
         try {
-            const parsingWapper = (rows) => {
-                return this.clientOptions.disable_parsing
-                    ? rows
-                    : reportOptions
-                        ? (0, parser_1.parse)({ results: rows, reportOptions })
-                        : (0, parser_1.parse)({ results: rows, gaqlString: gaqlQuery });
-            };
-            const { response, totalResultsCount } = await this.paginatedSearch(gaqlQuery, requestOptions, parsingWapper);
+            const { response, totalResultsCount } = await this.paginatedSearch(gaqlQuery, requestOptions);
             if (this.hooks.onQueryEnd && useHooks) {
                 const queryResolution = { resolved: false };
                 await this.hooks.onQueryEnd({
@@ -206,6 +300,7 @@ class Customer extends serviceFactory_1.default {
         }
     }
     async *streamer(gaqlQuery, requestOptions = {}, reportOptions) {
+        var _a;
         const baseHookArguments = {
             credentials: this.credentials,
             query: gaqlQuery,
@@ -229,58 +324,74 @@ class Customer extends serviceFactory_1.default {
                 return;
             }
         }
-        const { service, request } = this.buildSearchStreamRequestAndService(gaqlQuery, requestOptions);
-        const stream = service.searchStream(request, {
-            otherArgs: { headers: this.callHeaders },
-        });
-        let streamFinished = false;
-        const accumulator = [];
-        let nextChunk = (0, utils_1.createNextChunkArrivedPromise)();
-        stream.on("data", (chunk) => {
-            const results = chunk.summary_row ? [chunk.summary_row] : chunk.results;
-            const parsedResponse = this.clientOptions.disable_parsing
-                ? results
-                : reportOptions
-                    ? (0, parser_1.parse)({ results, reportOptions })
-                    : (0, parser_1.parse)({ results, gaqlString: gaqlQuery });
-            accumulator.push(...parsedResponse);
-            nextChunk.resolve();
-            nextChunk = (0, utils_1.createNextChunkArrivedPromise)();
-        });
-        stream.on("error", (searchError) => {
-            nextChunk.reject(searchError);
-        });
-        stream.on("end", () => {
-            streamFinished = true;
-            nextChunk.resolve();
-        });
         try {
-            while (!streamFinished || accumulator.length) {
-                if (accumulator.length > 0) {
-                    const item = accumulator.shift();
-                    if (item === undefined) {
-                        throw new Error("UNDEFINED_STREAM_ERROR");
-                    }
-                    yield item;
+            const accessToken = await this.getAccessToken();
+            const args = this.prepareGoogleAdsServicePostRequestArgs("searchStream", accessToken, {
+                responseType: "stream",
+                data: {
+                    query: gaqlQuery,
+                    ...requestOptions,
+                },
+            });
+            const response = await (0, axios_1.default)(args);
+            const stream = response.data;
+            // The options below help to make the stream less CPU intensive.
+            const parser = new stream_json_1.Parser({
+                streamValues: false,
+                streamKeys: false,
+                packValues: true,
+                packKeys: true,
+            });
+            const pipeline = (0, stream_chain_1.chain)([stream, parser, (0, StreamArray_1.streamArray)()]);
+            let count = 0;
+            for await (const data of pipeline) {
+                const results = (_a = data.value.results) !== null && _a !== void 0 ? _a : [data.value.summaryRow];
+                count += results.length;
+                if (this.clientOptions.max_reporting_rows &&
+                    count > this.clientOptions.max_reporting_rows &&
+                    !this.gaqlQueryStringIncludesLimit(gaqlQuery)) {
+                    throw this.generateTooManyRowsError();
                 }
-                else {
-                    await nextChunk.newPromise;
+                for (const row of results) {
+                    const parsed = this.decamelizeKeysIfNeeded(row);
+                    yield parsed;
                 }
             }
+            return;
         }
-        catch (searchError) {
-            const googleAdsError = this.getGoogleAdsError(searchError);
-            if (this.hooks.onStreamError) {
-                await this.hooks.onStreamError({
-                    ...baseHookArguments,
-                    error: googleAdsError,
-                });
+        catch (e) {
+            try {
+                await this.handleStreamError(e);
+            }
+            catch (_e) {
+                if (this.hooks.onStreamError) {
+                    await this.hooks.onStreamError({
+                        ...baseHookArguments,
+                        error: _e,
+                    });
+                }
+                throw _e;
             }
-            throw googleAdsError;
         }
-        finally {
-            stream.destroy();
+    }
+    async handleStreamError(e) {
+        var _a;
+        if (!((_a = e === null || e === void 0 ? void 0 : e.response) === null || _a === void 0 ? void 0 : _a.data)) {
+            throw e;
         }
+        // The error is a stream, so some effort is required to parse it.
+        const stream = e.response.data;
+        const pipeline = (0, stream_chain_1.chain)([stream, (0, stream_json_1.parser)(), (0, StreamArray_1.streamArray)()]);
+        let googleAdsFailure;
+        // Only throw the first error.
+        pipeline.once("data", (data) => {
+            googleAdsFailure = new protos_1.errors.GoogleAdsFailure(this.decamelizeKeysIfNeeded(data.value.error.details[0]));
+        });
+        // Must always reject.
+        await new Promise((_, reject) => {
+            pipeline.on("end", () => reject(googleAdsFailure));
+            pipeline.on("error", (err) => reject(err));
+        });
     }
     /**
      * @description Creates, updates, or removes resources. This method supports atomic transactions
@@ -360,5 +471,31 @@ class Customer extends serviceFactory_1.default {
             },
         };
     }
+    prepareGoogleAdsServicePostRequestArgs(functionName, accessToken, extra) {
+        var _a, _b;
+        return {
+            method: "POST",
+            url: `https://googleads.googleapis.com/${version_1.googleAdsVersion}/customers/${this.customerOptions.customer_id}/googleAds:${functionName}`,
+            headers: {
+                Authorization: `Bearer ${accessToken}`,
+                ...this.callHeaders,
+                "developer-token": (_a = this.clientOptions.developer_token) !== null && _a !== void 0 ? _a : "",
+                "login-customer-id": (_b = this.credentials.login_customer_id) !== null && _b !== void 0 ? _b : "",
+            },
+            ...extra,
+        };
+    }
+    decamelizeKeysIfNeeded(input) {
+        if (this.clientOptions.disable_parsing) {
+            return input;
+        }
+        return (0, parserRest_1.decamelizeKeys)(input);
+    }
+    gaqlQueryStringIncludesLimit(gaqlQuery) {
+        return gaqlQuery.toLowerCase().includes("limit ");
+    }
+    generateTooManyRowsError() {
+        return new Error(`Exceeded the maximum number of rows set by "max_reporting_rows" (${this.clientOptions.max_reporting_rows}).`);
+    }
 }
 exports.Customer = Customer;

package/build/src/parserRest.d.ts

@@ -0,0 +1,4 @@
+/**
+ * JSON Rest parsing
+ */
+export declare const decamelizeKeys: (input: any) => any;

package/build/src/parserRest.js

@@ -0,0 +1,88 @@
+"use strict";
+/**
+ * JSON Rest parsing
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.decamelizeKeys = void 0;
+const map_obj_1 = __importDefault(require("map-obj"));
+const circ_json_1 = require("circ-json");
+const utils_1 = require("./utils");
+const fields_1 = require("./protos/autogen/fields");
+const enums_1 = require("./protos/autogen/enums");
+const fieldDataTypes = (0, circ_json_1.parse)(fields_1.fieldDataTypes);
+const decamelizeCache = new Map();
+const fieldTypeCache = new Map();
+const isObject = (value) => typeof value === "object" && value !== null;
+const decamelizeKeys = (input) => {
+    if (!isObject(input)) {
+        return input;
+    }
+    const makeMapper = (parentPath) => (key, value) => {
+        key = cachedDecamelize(key);
+        if (isObject(value)) {
+            const path = parentPath === undefined ? key : `${parentPath}.${key}`;
+            // @ts-ignore
+            value = (0, map_obj_1.default)(value, makeMapper(path));
+        }
+        else {
+            value = cachedValueParser(key, parentPath, value);
+        }
+        return [key, value];
+    };
+    // @ts-ignore
+    return (0, map_obj_1.default)(input, makeMapper());
+};
+exports.decamelizeKeys = decamelizeKeys;
+const cachedDecamelize = (key) => {
+    const cachedResult = decamelizeCache.get(key);
+    if (cachedResult) {
+        return cachedResult;
+    }
+    const newKey = (0, utils_1.toSnakeCase)(key);
+    decamelizeCache.set(key, newKey);
+    return newKey;
+};
+const cachedValueParser = (key, parentPath, value) => {
+    let newValue = value;
+    const fullPath = parentPath ? `${parentPath}.${key}` : key;
+    const megaDataType = getTypeFromPath(fullPath);
+    if (megaDataType === undefined && !fullPath.startsWith("@")) {
+        console.warn(`No data type found for ${fullPath}`);
+    }
+    else if (typeof megaDataType === "object") {
+        newValue = megaDataType[value];
+    }
+    else if (megaDataType === "INT64") {
+        newValue = Number(value);
+    }
+    else if (megaDataType === "ENUM") {
+        // Some enums aren't embedded in megaDataType, so we need this fallback.
+        // @ts-expect-error typescript doesn't like accessing items in a namespace with a string
+        newValue = enums_1.enums[fields_1.fields.enumFields[fullPath]][value]; // e.g. enums['CampaignStatus'][ENABLED] = "2"
+    }
+    return newValue;
+};
+const getTypeFromPath = (path) => {
+    const cachedResult = fieldTypeCache.get(path);
+    if (cachedResult) {
+        return cachedResult;
+    }
+    const t = get(fieldDataTypes, path);
+    fieldTypeCache.set(path, t);
+    return t;
+};
+// Copied from youmightnotneed.com
+const get = (obj, path) => {
+    if (!path)
+        return undefined;
+    // Check if path is string or array. Regex : ensure that we do not have '.' and brackets.
+    // Regex explained: https://regexr.com/58j0k
+    const pathArray = path.match(/([^[.\]])+/g);
+    if (!pathArray)
+        return undefined;
+    // Find value
+    return pathArray.reduce((prevObj, key) => prevObj && prevObj[key], obj);
+};