@bytebrand/export-cars 1.26.26 → 1.26.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/README.md +28 -0
  2. package/dist/configs/config.d.ts +2 -1
  3. package/dist/configs/config.js +1 -0
  4. package/dist/configs/config.js.map +1 -1
  5. package/dist/configs/google.config.js +16 -0
  6. package/dist/configs/google.config.js.map +1 -1
  7. package/dist/exporters/google.exporter.js +12 -2
  8. package/dist/exporters/google.exporter.js.map +1 -1
  9. package/dist/interfaces/params.interface.d.ts +10 -0
  10. package/dist/services/publication-gate.d.ts +15 -0
  11. package/dist/services/publication-gate.js +107 -0
  12. package/dist/services/publication-gate.js.map +1 -0
  13. package/dist/transformers/google-feed-envkv.d.ts +2 -0
  14. package/dist/transformers/google-feed-envkv.js +62 -24
  15. package/dist/transformers/google-feed-envkv.js.map +1 -1
  16. package/dist/transformers/google-feed-item-row.d.ts +6 -1
  17. package/dist/transformers/google-feed-item-row.js +3 -1
  18. package/dist/transformers/google-feed-item-row.js.map +1 -1
  19. package/dist/tsconfig.build.tsbuildinfo +1 -1
  20. package/package.json +2 -2
  21. package/schema/feed-items.json +12 -0
  22. package/src/configs/config.ts +3 -1
  23. package/src/configs/google.config.ts +41 -26
  24. package/src/export.command.ts +38 -38
  25. package/src/exporters/google.exporter.ts +67 -56
  26. package/src/interfaces/params.interface.ts +43 -29
  27. package/src/services/feed-items-bigquery.writer.ts +101 -101
  28. package/src/services/publication-gate.ts +119 -0
  29. package/src/transformers/google-feed-envkv.ts +81 -25
  30. package/src/transformers/google-feed-item-row.ts +78 -71
  31. package/test/google-feed-envkv.test.ts +8 -3
  32. package/test/google-feed-item-row.test.ts +135 -121
  33. package/test/publication-gate.test.ts +86 -0
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bytebrand/export-cars",
3
- "version": "1.26.26",
3
+ "version": "1.26.28",
4
4
  "description": "",
5
5
  "author": "bytebrand",
6
6
  "license": "UNLICENSED",
@@ -11,7 +11,7 @@
11
11
  "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
12
12
  "lint": "eslint -c .eslintrc.js --ext .ts src/",
13
13
  "start:dev": "nest start --watch",
14
- "test": "ts-node --transpile-only test/google-feed-envkv.test.ts && ts-node --transpile-only test/google-feed-item-row.test.ts"
14
+ "test": "ts-node --transpile-only test/google-feed-envkv.test.ts && ts-node --transpile-only test/google-feed-item-row.test.ts && ts-node --transpile-only test/publication-gate.test.ts"
15
15
  },
16
16
  "dependencies": {
17
17
  "@bytebrand/autode-request": "^1.19.2",
@@ -118,5 +118,17 @@
118
118
  "type": "STRING",
119
119
  "mode": "NULLABLE",
120
120
  "description": "<g:availability> as sent"
121
+ },
122
+ {
123
+ "name": "publicationGate",
124
+ "type": "STRING",
125
+ "mode": "NULLABLE",
126
+ "description": "EnVKV C8 gate mode of the run (audit / enforce), null when off"
127
+ },
128
+ {
129
+ "name": "estimatedAttributes",
130
+ "type": "STRING",
131
+ "mode": "NULLABLE",
132
+ "description": "Comma-separated EnVKV attributes whose value CBD estimated; left out of the item in enforce mode"
121
133
  }
122
134
  ]
@@ -1,5 +1,5 @@
1
1
  import { Environment, ExportType, ImageType } from '../enums';
2
- import { FeedItemsBigQueryTarget, Params } from '../interfaces/params.interface';
2
+ import { FeedItemsBigQueryTarget, Params, PublicationGateConfig } from '../interfaces/params.interface';
3
3
 
4
4
  export class Config {
5
5
  apikey: string;
@@ -43,6 +43,8 @@ export class Config {
43
43
  runId?: string;
44
44
  // Set only when the job configures the feed items BigQuery table; unset = no BigQuery calls.
45
45
  feedItemsBigQuery?: FeedItemsBigQueryTarget;
46
+ // EnVKV C8 publication gate of the Google feed; unset = off.
47
+ publicationGate?: PublicationGateConfig;
46
48
 
47
49
  protected environmentConfig = {
48
50
  [Environment.DEVELOP]: {
@@ -1,26 +1,41 @@
1
- import { ExportType, FileType, ImageType } from 'src/enums';
2
- import { resolveMaxFeedSalesPrice } from '../constants/feed-price.constants';
3
- import { Params } from '../interfaces/params.interface';
4
- import { Config } from './config';
5
-
6
- const trimmed = (value: unknown): string => (typeof value === 'string' ? value.trim() : '');
7
-
8
- export class GoogleConfig extends Config {
9
- constructor(params: Params) {
10
- super(params);
11
- this.pathesConfig.localConvertedDirectory = `resources/${ExportType.GOOGLE}-export`;
12
- this.imageType = ImageType.default;
13
- this.maxCurrentSalesPrice = resolveMaxFeedSalesPrice(params.maxCurrentSalesPrice);
14
- const feedItemsBigQuery = {
15
- project: trimmed(params.feedItemsBqProject),
16
- dataset: trimmed(params.feedItemsBqDataset),
17
- table: trimmed(params.feedItemsBqTable),
18
- };
19
- if (feedItemsBigQuery.project && feedItemsBigQuery.dataset && feedItemsBigQuery.table) {
20
- this.feedItemsBigQuery = feedItemsBigQuery;
21
- }
22
- this.outputFiles = [
23
- `${this.pathesConfig.localConvertedDirectory}/api_export_cars_${ExportType.GOOGLE}.${FileType.XML}`,
24
- ];
25
- }
26
- }
1
+ import { ExportType, FileType, ImageType } from 'src/enums';
2
+ import { resolveMaxFeedSalesPrice } from '../constants/feed-price.constants';
3
+ import { Params, PublicationGateConfig } from '../interfaces/params.interface';
4
+ import { Config } from './config';
5
+
6
+ const trimmed = (value: unknown): string => (typeof value === 'string' ? value.trim() : '');
7
+
8
+ export class GoogleConfig extends Config {
9
+ constructor(params: Params) {
10
+ super(params);
11
+ this.pathesConfig.localConvertedDirectory = `resources/${ExportType.GOOGLE}-export`;
12
+ this.imageType = ImageType.default;
13
+ this.maxCurrentSalesPrice = resolveMaxFeedSalesPrice(params.maxCurrentSalesPrice);
14
+ const feedItemsBigQuery = {
15
+ project: trimmed(params.feedItemsBqProject),
16
+ dataset: trimmed(params.feedItemsBqDataset),
17
+ table: trimmed(params.feedItemsBqTable),
18
+ };
19
+ if (feedItemsBigQuery.project && feedItemsBigQuery.dataset && feedItemsBigQuery.table) {
20
+ this.feedItemsBigQuery = feedItemsBigQuery;
21
+ }
22
+ const gateMode = trimmed(params.publicationGate).toLowerCase();
23
+ if (gateMode === 'audit' || gateMode === 'enforce') {
24
+ const gate = {
25
+ mode: gateMode as PublicationGateConfig['mode'],
26
+ project: trimmed(params.publicationGateBqProject),
27
+ dataset: trimmed(params.publicationGateBqDataset),
28
+ table: trimmed(params.publicationGateBqTable),
29
+ };
30
+ if (!gate.project || !gate.dataset || !gate.table) {
31
+ throw new Error(`publicationGate=${gateMode} needs publicationGateBqProject, publicationGateBqDataset and publicationGateBqTable`);
32
+ }
33
+ this.publicationGate = gate;
34
+ } else if (gateMode && gateMode !== 'off') {
35
+ throw new Error(`publicationGate must be off, audit or enforce, got "${gateMode}"`);
36
+ }
37
+ this.outputFiles = [
38
+ `${this.pathesConfig.localConvertedDirectory}/api_export_cars_${ExportType.GOOGLE}.${FileType.XML}`,
39
+ ];
40
+ }
41
+ }
@@ -1,38 +1,38 @@
1
- import { createInterface } from 'readline';
2
- import { Command, CommandRunner } from 'nest-commander';
3
- import { ConfigService } from './services';
4
- import { Input } from './interfaces/input.interface';
5
- import { exportSchema } from './export.schema';
6
- import { ModuleRef } from '@nestjs/core';
7
- import { Exporter } from './interfaces/exporter.interface';
8
- import { LoggerService } from '@bytebrand/cronicle-helper';
9
-
10
- @Command({
11
- name: 'export',
12
- options: { isDefault: true },
13
- })
14
- export class ExportRunner extends CommandRunner {
15
- constructor(
16
- private configService: ConfigService,
17
- private moduleRef: ModuleRef,
18
- ) {
19
- super();
20
- }
21
-
22
- async run(): Promise<void> {
23
- const rl = createInterface({ input: process.stdin });
24
- await new Promise<void>((resolve, reject) => {
25
- rl.on('line', async (data: string) => {
26
- const input = JSON.parse(data) as Input;
27
- LoggerService.log(`Running job: ${input.id} - ${JSON.stringify(input)} \n`);
28
-
29
- this.configService.config = new exportSchema[input.params.exportType].config(input.params);
30
- this.configService.config.runId = input.id;
31
- const exporter = await this.moduleRef.create<Exporter>(exportSchema[input.params.exportType].exporter);
32
- await exporter.export().catch(reject);
33
-
34
- resolve(rl.close());
35
- });
36
- });
37
- }
38
- }
1
+ import { createInterface } from 'readline';
2
+ import { Command, CommandRunner } from 'nest-commander';
3
+ import { ConfigService } from './services';
4
+ import { Input } from './interfaces/input.interface';
5
+ import { exportSchema } from './export.schema';
6
+ import { ModuleRef } from '@nestjs/core';
7
+ import { Exporter } from './interfaces/exporter.interface';
8
+ import { LoggerService } from '@bytebrand/cronicle-helper';
9
+
10
+ @Command({
11
+ name: 'export',
12
+ options: { isDefault: true },
13
+ })
14
+ export class ExportRunner extends CommandRunner {
15
+ constructor(
16
+ private configService: ConfigService,
17
+ private moduleRef: ModuleRef,
18
+ ) {
19
+ super();
20
+ }
21
+
22
+ async run(): Promise<void> {
23
+ const rl = createInterface({ input: process.stdin });
24
+ await new Promise<void>((resolve, reject) => {
25
+ rl.on('line', async (data: string) => {
26
+ const input = JSON.parse(data) as Input;
27
+ LoggerService.log(`Running job: ${input.id} - ${JSON.stringify(input)} \n`);
28
+
29
+ this.configService.config = new exportSchema[input.params.exportType].config(input.params);
30
+ this.configService.config.runId = input.id;
31
+ const exporter = await this.moduleRef.create<Exporter>(exportSchema[input.params.exportType].exporter);
32
+ await exporter.export().catch(reject);
33
+
34
+ resolve(rl.close());
35
+ });
36
+ });
37
+ }
38
+ }
@@ -1,56 +1,67 @@
1
- import { Injectable } from '@nestjs/common';
2
- import { ModuleRef } from '@nestjs/core';
3
-
4
- import { CronicleUtils } from '@bytebrand/cronicle-helper';
5
- import { FsChunkInterface } from '@bytebrand/chunkable';
6
-
7
- import { CarService, ConfigService, LoaderService, RulesService } from '../services';
8
- import { GoogleTransformer } from '../transformers/google.transformer';
9
- import { buildGoogleFeedItemRow } from '../transformers/google-feed-item-row';
10
- import { FeedItemsBigQueryWriter } from '../services/feed-items-bigquery.writer';
11
- import { GoogleXmlBuilder } from '../formats/google-xml.builder';
12
- import { Exporter } from '../interfaces/exporter.interface';
13
-
14
- @Injectable()
15
- export class GoogleExporter implements Exporter {
16
- constructor(
17
- private carService: CarService,
18
- private configService: ConfigService,
19
- private rulesService: RulesService,
20
- private loaderService: LoaderService,
21
- private moduleRef: ModuleRef,
22
- ) { }
23
-
24
- async export(): Promise<void> {
25
- const [fileName] = this.configService.config.outputFiles;
26
- await CronicleUtils.ensureDir(this.configService.config.pathesConfig.localConvertedDirectory);
27
- await CronicleUtils.emptyDir(this.configService.config.pathesConfig.localConvertedDirectory);
28
-
29
- const googleXmlBuilder = new GoogleXmlBuilder(fileName);
30
- await googleXmlBuilder.buildStart(this.configService.config.domain);
31
-
32
- const googleTransformer = await this.moduleRef.create(GoogleTransformer);
33
-
34
- // Opt-in audit of the sent items (feedItemsBq* job params); null = no BigQuery calls at all.
35
- const { feedItemsBigQuery, runId } = this.configService.config;
36
- const feedItemsWriter = feedItemsBigQuery ? new FeedItemsBigQueryWriter(feedItemsBigQuery) : null;
37
- const exportedAt = new Date();
38
-
39
- await this.loaderService.process(async (chunk: FsChunkInterface) => {
40
- chunk.data = chunk.data.filter(car => this.carService.filterCar(car, this.rulesService.rules));
41
-
42
- for (const data of chunk.data) {
43
- const car = await googleTransformer.transform(data);
44
- if (car) {
45
- await googleXmlBuilder.addItem(car);
46
- feedItemsWriter?.add(buildGoogleFeedItemRow(data.car, car, runId, exportedAt));
47
- }
48
- this.carService.counter.countCar(data.images);
49
- }
50
- });
51
-
52
- await googleXmlBuilder.buildEnd();
53
- await feedItemsWriter?.close();
54
- console.log(this.carService.counter.getCountedCars());
55
- }
56
- }
1
+ import { Injectable } from '@nestjs/common';
2
+ import { ModuleRef } from '@nestjs/core';
3
+
4
+ import { CronicleUtils } from '@bytebrand/cronicle-helper';
5
+ import { FsChunkInterface } from '@bytebrand/chunkable';
6
+
7
+ import { CarService, ConfigService, LoaderService, RulesService } from '../services';
8
+ import { GoogleTransformer } from '../transformers/google.transformer';
9
+ import { buildGoogleFeedItemRow } from '../transformers/google-feed-item-row';
10
+ import { FeedItemsBigQueryWriter } from '../services/feed-items-bigquery.writer';
11
+ import { loadPublicationGate } from '../services/publication-gate';
12
+ import { GoogleXmlBuilder } from '../formats/google-xml.builder';
13
+ import { Exporter } from '../interfaces/exporter.interface';
14
+
15
+ @Injectable()
16
+ export class GoogleExporter implements Exporter {
17
+ constructor(
18
+ private carService: CarService,
19
+ private configService: ConfigService,
20
+ private rulesService: RulesService,
21
+ private loaderService: LoaderService,
22
+ private moduleRef: ModuleRef,
23
+ ) { }
24
+
25
+ async export(): Promise<void> {
26
+ // EnVKV C8: estimated EnVKV values (CBD enrichment log); off unless the job sets publicationGate.
27
+ // Read first: in enforce mode a failure stops the run before the output directory is emptied.
28
+ const gate = await loadPublicationGate(this.configService.config.publicationGate);
29
+ const [fileName] = this.configService.config.outputFiles;
30
+ await CronicleUtils.ensureDir(this.configService.config.pathesConfig.localConvertedDirectory);
31
+ await CronicleUtils.emptyDir(this.configService.config.pathesConfig.localConvertedDirectory);
32
+
33
+ const googleXmlBuilder = new GoogleXmlBuilder(fileName);
34
+ await googleXmlBuilder.buildStart(this.configService.config.domain);
35
+
36
+ const googleTransformer = await this.moduleRef.create(GoogleTransformer);
37
+
38
+ // Opt-in audit of the sent items (feedItemsBq* job params); null = no BigQuery calls at all.
39
+ const { feedItemsBigQuery, runId } = this.configService.config;
40
+ const feedItemsWriter = feedItemsBigQuery ? new FeedItemsBigQueryWriter(feedItemsBigQuery) : null;
41
+ const exportedAt = new Date();
42
+ let gatedItems = 0;
43
+
44
+ await this.loaderService.process(async (chunk: FsChunkInterface) => {
45
+ chunk.data = chunk.data.filter(car => this.carService.filterCar(car, this.rulesService.rules));
46
+
47
+ for (const data of chunk.data) {
48
+ const car = await googleTransformer.transform(data);
49
+ if (car) {
50
+ const estimatedAttributes = gate.estimatedAttributes(data.car);
51
+ if (estimatedAttributes.length) gatedItems += 1;
52
+ await googleXmlBuilder.addItem(gate.apply(car, estimatedAttributes));
53
+ const gateInfo = gate.mode === 'off' ? null : { mode: gate.mode, estimatedAttributes };
54
+ feedItemsWriter?.add(buildGoogleFeedItemRow(data.car, car, runId, exportedAt, gateInfo));
55
+ }
56
+ this.carService.counter.countCar(data.images);
57
+ }
58
+ });
59
+
60
+ await googleXmlBuilder.buildEnd();
61
+ if (gate.mode !== 'off') {
62
+ console.log(`Publication gate ${gate.mode}: ${gatedItems} items with estimated EnVKV values${gate.mode === 'enforce' ? ' (attributes left out)' : ''}`);
63
+ }
64
+ await feedItemsWriter?.close();
65
+ console.log(this.carService.counter.getCountedCars());
66
+ }
67
+ }
@@ -1,29 +1,43 @@
1
- import { Environment, ExportType } from '../enums';
2
-
3
- export interface Params {
4
- environment: Environment;
5
- exportType?: ExportType;
6
- apikey?: string;
7
- String?: string;
8
- Number?: string;
9
- Date?: string;
10
- isNewCar?: boolean;
11
- cpc?: string;
12
- ownerId?: string;
13
- metaPushPriorityCsvUrl?: string;
14
- // Cronicle plugin param (text field, so it arrives as a string); overrides the
15
- // default MAX_FEED_SALES_PRICE_EUR cap in the configs that apply one.
16
- maxCurrentSalesPrice?: string;
17
- // Google feed only, opt-in: when all three are set, every exported item is also written to
18
- // `<project>.<dataset>.<table>` in BigQuery (schema/feed-items.json, see README).
19
- feedItemsBqProject?: string;
20
- feedItemsBqDataset?: string;
21
- feedItemsBqTable?: string;
22
- }
23
-
24
- // Parsed from feedItemsBq* by GoogleConfig.
25
- export interface FeedItemsBigQueryTarget {
26
- project: string;
27
- dataset: string;
28
- table: string;
29
- }
1
+ import { Environment, ExportType } from '../enums';
2
+
3
+ export interface Params {
4
+ environment: Environment;
5
+ exportType?: ExportType;
6
+ apikey?: string;
7
+ String?: string;
8
+ Number?: string;
9
+ Date?: string;
10
+ isNewCar?: boolean;
11
+ cpc?: string;
12
+ ownerId?: string;
13
+ metaPushPriorityCsvUrl?: string;
14
+ // Cronicle plugin param (text field, so it arrives as a string); overrides the
15
+ // default MAX_FEED_SALES_PRICE_EUR cap in the configs that apply one.
16
+ maxCurrentSalesPrice?: string;
17
+ // Google feed only, opt-in: when all three are set, every exported item is also written to
18
+ // `<project>.<dataset>.<table>` in BigQuery (schema/feed-items.json, see README).
19
+ feedItemsBqProject?: string;
20
+ feedItemsBqDataset?: string;
21
+ feedItemsBqTable?: string;
22
+ // Google feed only, EnVKV C8: off (default) | audit | enforce, and the car-basic-data enrichment log
23
+ // table the estimates are read from (see services/publication-gate.ts).
24
+ publicationGate?: string;
25
+ publicationGateBqProject?: string;
26
+ publicationGateBqDataset?: string;
27
+ publicationGateBqTable?: string;
28
+ }
29
+
30
+ // Parsed from feedItemsBq* by GoogleConfig.
31
+ export interface FeedItemsBigQueryTarget {
32
+ project: string;
33
+ dataset: string;
34
+ table: string;
35
+ }
36
+
37
+ // Parsed from publicationGate* by GoogleConfig.
38
+ export interface PublicationGateConfig {
39
+ mode: 'off' | 'audit' | 'enforce';
40
+ project: string;
41
+ dataset: string;
42
+ table: string;
43
+ }
@@ -1,101 +1,101 @@
1
- import { join } from 'path';
2
- import { BigQueryLogger } from '@bytebrand/bigquery-logger';
3
- import { FeedItemsBigQueryTarget } from '../interfaces/params.interface';
4
- import { FeedItemRow } from '../transformers/google-feed-item-row';
5
-
6
- const BATCH_SIZE = 500;
7
- // After this many failed batches the writer gives up for the run (missing table, bad credentials).
8
- const MAX_FAILED_BATCHES = 3;
9
- const CLOSE_TIMEOUT_MS = 60000;
10
-
11
- class FeedItemsBigQueryLogger extends BigQueryLogger {
12
- async insertRows(rows: FeedItemRow[]): Promise<void> {
13
- await this.client
14
- .dataset(this.options.BigQueryDataset)
15
- .table(this.options.BigQueryLogTable)
16
- .insert(rows, { skipInvalidRows: true, ignoreUnknownValues: true });
17
- }
18
- }
19
-
20
- // Audit trail of the feed items: rows are buffered and inserted in batches of 500 in the background,
21
- // one batch at a time. Errors are logged and swallowed — the feed export must never fail because of
22
- // BigQuery.
23
- export class FeedItemsBigQueryWriter {
24
- private logger: FeedItemsBigQueryLogger;
25
- private buffer: FeedItemRow[] = [];
26
- private queue: Promise<void> = Promise.resolve();
27
- private failedBatches = 0;
28
- private insertedRows = 0;
29
-
30
- constructor(private target: FeedItemsBigQueryTarget) {
31
- try {
32
- const credentials = require(join(process.cwd(), 'credentials/google-cloud-project-id.json'));
33
- this.logger = new FeedItemsBigQueryLogger(credentials, {
34
- BigQueryProject: target.project,
35
- BigQueryDataset: target.dataset,
36
- BigQueryLogTable: target.table,
37
- });
38
- } catch (e) {
39
- // First line only: a missing credentials file comes with the whole require stack.
40
- console.log(`Feed items BigQuery disabled: ${String(e?.message || e).split('\n')[0]}`);
41
- }
42
- }
43
-
44
- add(row: FeedItemRow): void {
45
- if (!this.enabled) return;
46
- this.buffer.push(row);
47
- if (this.buffer.length >= BATCH_SIZE) this.flush();
48
- }
49
-
50
- // Waits for the pending batches, but never longer than CLOSE_TIMEOUT_MS: a hanging BigQuery
51
- // must not hold up the feed export (the XML is already written at this point).
52
- async close(): Promise<void> {
53
- this.flush();
54
- let timer: NodeJS.Timeout | undefined;
55
- const timeout = new Promise<'timeout'>((resolve) => {
56
- timer = setTimeout(() => resolve('timeout'), CLOSE_TIMEOUT_MS);
57
- });
58
- const outcome = await Promise.race([this.queue.then(() => 'done' as const), timeout]);
59
- if (timer) clearTimeout(timer);
60
- if (outcome === 'timeout') console.error(`Feed items BigQuery: gave up waiting after ${CLOSE_TIMEOUT_MS} ms`);
61
- if (this.logger) {
62
- console.log(`Feed items BigQuery: ${this.insertedRows} rows to ${this.tableName}, ${this.failedBatches} failed batches`);
63
- }
64
- }
65
-
66
- private get enabled(): boolean {
67
- return !!this.logger && this.failedBatches < MAX_FAILED_BATCHES;
68
- }
69
-
70
- private get tableName(): string {
71
- return `${this.target.project}.${this.target.dataset}.${this.target.table}`;
72
- }
73
-
74
- private flush(): void {
75
- if (!this.buffer.length) return;
76
- const rows = this.buffer;
77
- this.buffer = [];
78
- this.queue = this.queue.then(() => this.insert(rows));
79
- }
80
-
81
- private async insert(rows: FeedItemRow[]): Promise<void> {
82
- if (!this.enabled) return;
83
- try {
84
- await this.logger.insertRows(rows);
85
- this.insertedRows += rows.length;
86
- } catch (e) {
87
- // PartialFailureError: the valid rows went in (skipInvalidRows), `errors` lists the rejected
88
- // ones; the first is enough to see the cause.
89
- const rejected: any[] = e?.name === 'PartialFailureError' && Array.isArray(e.errors) ? e.errors : null;
90
- const firstRowError = rejected?.[0]?.errors?.[0];
91
- const detail = firstRowError ? ` (${firstRowError.reason}: ${firstRowError.message})` : '';
92
- if (rejected) {
93
- this.insertedRows += rows.length - rejected.length;
94
- console.log(`Feed items BigQuery rejected ${rejected.length} of ${rows.length} rows${detail}`);
95
- } else {
96
- this.failedBatches++;
97
- console.log(`Feed items BigQuery insert of ${rows.length} rows failed: ${e?.message || e}`);
98
- }
99
- }
100
- }
101
- }
1
+ import { join } from 'path';
2
+ import { BigQueryLogger } from '@bytebrand/bigquery-logger';
3
+ import { FeedItemsBigQueryTarget } from '../interfaces/params.interface';
4
+ import { FeedItemRow } from '../transformers/google-feed-item-row';
5
+
6
+ const BATCH_SIZE = 500;
7
+ // After this many failed batches the writer gives up for the run (missing table, bad credentials).
8
+ const MAX_FAILED_BATCHES = 3;
9
+ const CLOSE_TIMEOUT_MS = 60000;
10
+
11
+ class FeedItemsBigQueryLogger extends BigQueryLogger {
12
+ async insertRows(rows: FeedItemRow[]): Promise<void> {
13
+ await this.client
14
+ .dataset(this.options.BigQueryDataset)
15
+ .table(this.options.BigQueryLogTable)
16
+ .insert(rows, { skipInvalidRows: true, ignoreUnknownValues: true });
17
+ }
18
+ }
19
+
20
+ // Audit trail of the feed items: rows are buffered and inserted in batches of 500 in the background,
21
+ // one batch at a time. Errors are logged and swallowed — the feed export must never fail because of
22
+ // BigQuery.
23
+ export class FeedItemsBigQueryWriter {
24
+ private logger: FeedItemsBigQueryLogger;
25
+ private buffer: FeedItemRow[] = [];
26
+ private queue: Promise<void> = Promise.resolve();
27
+ private failedBatches = 0;
28
+ private insertedRows = 0;
29
+
30
+ constructor(private target: FeedItemsBigQueryTarget) {
31
+ try {
32
+ const credentials = require(join(process.cwd(), 'credentials/google-cloud-project-id.json'));
33
+ this.logger = new FeedItemsBigQueryLogger(credentials, {
34
+ BigQueryProject: target.project,
35
+ BigQueryDataset: target.dataset,
36
+ BigQueryLogTable: target.table,
37
+ });
38
+ } catch (e) {
39
+ // First line only: a missing credentials file comes with the whole require stack.
40
+ console.log(`Feed items BigQuery disabled: ${String(e?.message || e).split('\n')[0]}`);
41
+ }
42
+ }
43
+
44
+ add(row: FeedItemRow): void {
45
+ if (!this.enabled) return;
46
+ this.buffer.push(row);
47
+ if (this.buffer.length >= BATCH_SIZE) this.flush();
48
+ }
49
+
50
+ // Waits for the pending batches, but never longer than CLOSE_TIMEOUT_MS: a hanging BigQuery
51
+ // must not hold up the feed export (the XML is already written at this point).
52
+ async close(): Promise<void> {
53
+ this.flush();
54
+ let timer: NodeJS.Timeout | undefined;
55
+ const timeout = new Promise<'timeout'>((resolve) => {
56
+ timer = setTimeout(() => resolve('timeout'), CLOSE_TIMEOUT_MS);
57
+ });
58
+ const outcome = await Promise.race([this.queue.then(() => 'done' as const), timeout]);
59
+ if (timer) clearTimeout(timer);
60
+ if (outcome === 'timeout') console.error(`Feed items BigQuery: gave up waiting after ${CLOSE_TIMEOUT_MS} ms`);
61
+ if (this.logger) {
62
+ console.log(`Feed items BigQuery: ${this.insertedRows} rows to ${this.tableName}, ${this.failedBatches} failed batches`);
63
+ }
64
+ }
65
+
66
+ private get enabled(): boolean {
67
+ return !!this.logger && this.failedBatches < MAX_FAILED_BATCHES;
68
+ }
69
+
70
+ private get tableName(): string {
71
+ return `${this.target.project}.${this.target.dataset}.${this.target.table}`;
72
+ }
73
+
74
+ private flush(): void {
75
+ if (!this.buffer.length) return;
76
+ const rows = this.buffer;
77
+ this.buffer = [];
78
+ this.queue = this.queue.then(() => this.insert(rows));
79
+ }
80
+
81
+ private async insert(rows: FeedItemRow[]): Promise<void> {
82
+ if (!this.enabled) return;
83
+ try {
84
+ await this.logger.insertRows(rows);
85
+ this.insertedRows += rows.length;
86
+ } catch (e) {
87
+ // PartialFailureError: the valid rows went in (skipInvalidRows), `errors` lists the rejected
88
+ // ones; the first is enough to see the cause.
89
+ const rejected: any[] = e?.name === 'PartialFailureError' && Array.isArray(e.errors) ? e.errors : null;
90
+ const firstRowError = rejected?.[0]?.errors?.[0];
91
+ const detail = firstRowError ? ` (${firstRowError.reason}: ${firstRowError.message})` : '';
92
+ if (rejected) {
93
+ this.insertedRows += rows.length - rejected.length;
94
+ console.log(`Feed items BigQuery rejected ${rejected.length} of ${rows.length} rows${detail}`);
95
+ } else {
96
+ this.failedBatches++;
97
+ console.log(`Feed items BigQuery insert of ${rows.length} rows failed: ${e?.message || e}`);
98
+ }
99
+ }
100
+ }
101
+ }