@attlaz/client 1.104.0 → 1.106.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/MarketPulse/Model/CrawlJobPage.d.ts +20 -0
- package/dist/MarketPulse/Model/CrawlJobPage.js +22 -0
- package/dist/MarketPulse/Model/StockStatus.d.ts +19 -0
- package/dist/MarketPulse/Model/StockStatus.js +21 -0
- package/dist/MarketPulse/Model/VendorProduct.d.ts +2 -1
- package/dist/MarketPulse/Model/VendorProduct.js +3 -2
- package/dist/MarketPulse/Model/VendorProductPriceHistoryEntry.d.ts +2 -1
- package/dist/MarketPulse/Model/VendorProductPriceHistoryEntry.js +3 -2
- package/dist/MarketPulse/Service/CrawlJobEndpoint.d.ts +11 -0
- package/dist/MarketPulse/Service/CrawlJobEndpoint.js +14 -0
- package/dist/MarketPulse/Service/VendorProductEndpoint.js +2 -2
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { ApiRecord } from '../../Model/ApiRecord.js';
|
|
2
|
+
/**
|
|
3
|
+
* One page of a crawl — `pulse.crawl_job_page`.
|
|
4
|
+
*
|
|
5
|
+
* A worker takes pages by claiming them, which is what `status` and `claimedAt` record: a claimed
|
|
6
|
+
* page is nobody else's to fetch until its owner finishes or its lease expires.
|
|
7
|
+
*/
|
|
8
|
+
export type CrawlJobPageStatus = 'pending' | 'running' | 'complete' | 'failed';
|
|
9
|
+
export declare class CrawlJobPage {
|
|
10
|
+
id: string;
|
|
11
|
+
crawlJobId: string;
|
|
12
|
+
url: string;
|
|
13
|
+
status: CrawlJobPageStatus;
|
|
14
|
+
/** When a worker took this page; its lease runs from here. */
|
|
15
|
+
claimedAt: Date | null;
|
|
16
|
+
/** How many times it has been claimed — a page released by an expired lease is retried. */
|
|
17
|
+
attempts: number;
|
|
18
|
+
constructor(id: string, crawlJobId: string, url: string);
|
|
19
|
+
static parse(raw: ApiRecord): CrawlJobPage;
|
|
20
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export class CrawlJobPage {
|
|
2
|
+
id;
|
|
3
|
+
crawlJobId;
|
|
4
|
+
url;
|
|
5
|
+
status = 'pending';
|
|
6
|
+
/** When a worker took this page; its lease runs from here. */
|
|
7
|
+
claimedAt = null;
|
|
8
|
+
/** How many times it has been claimed — a page released by an expired lease is retried. */
|
|
9
|
+
attempts = 0;
|
|
10
|
+
constructor(id, crawlJobId, url) {
|
|
11
|
+
this.id = id;
|
|
12
|
+
this.crawlJobId = crawlJobId;
|
|
13
|
+
this.url = url;
|
|
14
|
+
}
|
|
15
|
+
static parse(raw) {
|
|
16
|
+
const page = new CrawlJobPage(raw.id, raw.crawl_job_id ?? raw.crawl_job ?? '', raw.url ?? '');
|
|
17
|
+
page.status = (raw.status ?? 'pending');
|
|
18
|
+
page.claimedAt = raw.claimed_at === undefined || raw.claimed_at === null ? null : new Date(raw.claimed_at);
|
|
19
|
+
page.attempts = Number(raw.attempts ?? 0);
|
|
20
|
+
return page;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whether a vendor product is purchasable. Three states rather than a nullable boolean, because
|
|
3
|
+
* "we don't know" is a different fact from "it is out of stock" — most crawlers never report stock.
|
|
4
|
+
*
|
|
5
|
+
* Duplicated from apps-lib rather than imported, per the client's enum-duplication convention: the
|
|
6
|
+
* client must not depend on a server package.
|
|
7
|
+
*/
|
|
8
|
+
export type StockStatus = 'in_stock' | 'out_of_stock' | 'unknown';
|
|
9
|
+
/**
|
|
10
|
+
* Read `stock_status` from an API record. An unrecognised or absent value is `unknown`, never
|
|
11
|
+
* `out_of_stock` — guessing the latter would make a consumer hide a product that is probably on sale.
|
|
12
|
+
*/
|
|
13
|
+
export declare const parseStockStatus: (status: unknown) => StockStatus;
|
|
14
|
+
/**
|
|
15
|
+
* Convert a scraper's own boolean reading of stock into a status. Separate from parseStockStatus:
|
|
16
|
+
* that one reads the wire, this one is for callers holding a `boolean | null` of their own, where
|
|
17
|
+
* null genuinely means the source did not say.
|
|
18
|
+
*/
|
|
19
|
+
export declare const stockStatusFromBoolean: (inStock: boolean | null) => StockStatus;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read `stock_status` from an API record. An unrecognised or absent value is `unknown`, never
|
|
3
|
+
* `out_of_stock` — guessing the latter would make a consumer hide a product that is probably on sale.
|
|
4
|
+
*/
|
|
5
|
+
export const parseStockStatus = (status) => {
|
|
6
|
+
if (status === 'in_stock' || status === 'out_of_stock' || status === 'unknown') {
|
|
7
|
+
return status;
|
|
8
|
+
}
|
|
9
|
+
return 'unknown';
|
|
10
|
+
};
|
|
11
|
+
/**
|
|
12
|
+
* Convert a scraper's own boolean reading of stock into a status. Separate from parseStockStatus:
|
|
13
|
+
* that one reads the wire, this one is for callers holding a `boolean | null` of their own, where
|
|
14
|
+
* null genuinely means the source did not say.
|
|
15
|
+
*/
|
|
16
|
+
export const stockStatusFromBoolean = (inStock) => {
|
|
17
|
+
if (inStock === null) {
|
|
18
|
+
return 'unknown';
|
|
19
|
+
}
|
|
20
|
+
return inStock ? 'in_stock' : 'out_of_stock';
|
|
21
|
+
};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { StockStatus } from './StockStatus.js';
|
|
1
2
|
import { ApiRecord } from '../../Model/ApiRecord.js';
|
|
2
3
|
/**
|
|
3
4
|
* A competitor's product as tracked by Market Pulse — the `/pulse/vendors/:vendorId/products`
|
|
@@ -17,7 +18,7 @@ export declare class VendorProduct {
|
|
|
17
18
|
price: number | null;
|
|
18
19
|
originalPrice: number | null;
|
|
19
20
|
shippingCost: number | null;
|
|
20
|
-
|
|
21
|
+
stockStatus: StockStatus;
|
|
21
22
|
properties: Record<string, unknown> | null;
|
|
22
23
|
createdAt: Date | null;
|
|
23
24
|
updatedAt: Date | null;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { parseStockStatus } from './StockStatus.js';
|
|
1
2
|
/**
|
|
2
3
|
* A competitor's product as tracked by Market Pulse — the `/pulse/vendors/:vendorId/products`
|
|
3
4
|
* resource. `identifier` is the product's stable id on the competitor's site (the upsert key).
|
|
@@ -16,7 +17,7 @@ export class VendorProduct {
|
|
|
16
17
|
price = null;
|
|
17
18
|
originalPrice = null;
|
|
18
19
|
shippingCost = null;
|
|
19
|
-
|
|
20
|
+
stockStatus = 'unknown';
|
|
20
21
|
properties = null;
|
|
21
22
|
createdAt = null;
|
|
22
23
|
updatedAt = null;
|
|
@@ -46,7 +47,7 @@ export class VendorProduct {
|
|
|
46
47
|
product.price = toNumber(raw.price);
|
|
47
48
|
product.originalPrice = toNumber(raw.original_price);
|
|
48
49
|
product.shippingCost = toNumber(raw.shipping_cost);
|
|
49
|
-
product.
|
|
50
|
+
product.stockStatus = parseStockStatus(raw.stock_status);
|
|
50
51
|
product.properties = raw.properties ?? null;
|
|
51
52
|
product.createdAt = raw.created_at ? new Date(raw.created_at) : null;
|
|
52
53
|
product.updatedAt = raw.updated_at ? new Date(raw.updated_at) : null;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { StockStatus } from './StockStatus.js';
|
|
1
2
|
import { ApiRecord } from '../../Model/ApiRecord.js';
|
|
2
3
|
/**
|
|
3
4
|
* One recorded price point of a competitor product — the `/pulse/vendor-products/:id/price-history`
|
|
@@ -10,7 +11,7 @@ export declare class VendorProductPriceHistoryEntry {
|
|
|
10
11
|
price: number | null;
|
|
11
12
|
originalPrice: number | null;
|
|
12
13
|
shippingCost: number | null;
|
|
13
|
-
|
|
14
|
+
stockStatus: StockStatus;
|
|
14
15
|
isPromotion: boolean | null;
|
|
15
16
|
recordedAt: Date | null;
|
|
16
17
|
constructor(id: string, vendorProductId: string);
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { parseStockStatus } from './StockStatus.js';
|
|
1
2
|
/**
|
|
2
3
|
* One recorded price point of a competitor product — the `/pulse/vendor-products/:id/price-history`
|
|
3
4
|
* resource. Written when a crawl finds a price that differs from the last recorded one, so the
|
|
@@ -9,7 +10,7 @@ export class VendorProductPriceHistoryEntry {
|
|
|
9
10
|
price = null;
|
|
10
11
|
originalPrice = null;
|
|
11
12
|
shippingCost = null;
|
|
12
|
-
|
|
13
|
+
stockStatus = 'unknown';
|
|
13
14
|
isPromotion = null;
|
|
14
15
|
recordedAt = null;
|
|
15
16
|
constructor(id, vendorProductId) {
|
|
@@ -21,7 +22,7 @@ export class VendorProductPriceHistoryEntry {
|
|
|
21
22
|
entry.price = toNumber(raw.price);
|
|
22
23
|
entry.originalPrice = toNumber(raw.original_price);
|
|
23
24
|
entry.shippingCost = toNumber(raw.shipping_cost);
|
|
24
|
-
entry.
|
|
25
|
+
entry.stockStatus = parseStockStatus(raw.stock_status);
|
|
25
26
|
entry.isPromotion = typeof raw.is_promotion === 'boolean' ? raw.is_promotion : null;
|
|
26
27
|
entry.recordedAt = raw.recorded_at ? new Date(raw.recorded_at) : null;
|
|
27
28
|
return entry;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { CatalogId } from '../../Catalog/Model/CatalogId.js';
|
|
2
2
|
import { CursorPagination } from '../../Model/Pagination/CursorPagination.js';
|
|
3
3
|
import { CrawlJob, CrawlJobStatus } from '../Model/CrawlJob.js';
|
|
4
|
+
import { CrawlJobPage } from '../Model/CrawlJobPage.js';
|
|
4
5
|
import { CollectionResult } from '../../Model/Result/CollectionResult.js';
|
|
5
6
|
import { Endpoint } from '../../Service/Endpoint.js';
|
|
6
7
|
/**
|
|
@@ -32,6 +33,16 @@ export declare class CrawlJobEndpoint extends Endpoint {
|
|
|
32
33
|
markPagesParsed(crawlJobId: string, urls: string[]): Promise<void>;
|
|
33
34
|
/** Mark pages as failed (terminal, e.g. blocked/unfetchable). */
|
|
34
35
|
markPagesFailed(crawlJobId: string, urls: string[]): Promise<void>;
|
|
36
|
+
/**
|
|
37
|
+
* Atomically take up to `limit` of this crawl's pending pages to work on.
|
|
38
|
+
*
|
|
39
|
+
* Concurrent workers get disjoint sets, so a page is never fetched twice. An empty array means
|
|
40
|
+
* there is nothing left to do — that is how a worker knows to stop, not an error.
|
|
41
|
+
*
|
|
42
|
+
* Claiming starts a lease: finish the pages (mark them parsed or failed) or they are handed back
|
|
43
|
+
* to another worker once the lease expires.
|
|
44
|
+
*/
|
|
45
|
+
claimPages(crawlJobId: string, limit: number): Promise<CrawlJobPage[]>;
|
|
35
46
|
/** @param suffix a literal trailing segment, e.g. `/pages/crawled` — not caller data. */
|
|
36
47
|
private postUrls;
|
|
37
48
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { path } from '../../Http/Data/Path.js';
|
|
2
2
|
import { QueryString } from '../../Http/Data/QueryString.js';
|
|
3
3
|
import { CrawlJob } from '../Model/CrawlJob.js';
|
|
4
|
+
import { CrawlJobPage } from '../Model/CrawlJobPage.js';
|
|
4
5
|
import { Endpoint } from '../../Service/Endpoint.js';
|
|
5
6
|
/**
|
|
6
7
|
* Market Pulse crawl jobs — `/pulse/crawl-jobs`. Mirrors the PHP client's MarketPulse\CrawlJobEndpoint.
|
|
@@ -64,6 +65,19 @@ export class CrawlJobEndpoint extends Endpoint {
|
|
|
64
65
|
async markPagesFailed(crawlJobId, urls) {
|
|
65
66
|
await this.postUrls(crawlJobId, '/pages/failed', urls);
|
|
66
67
|
}
|
|
68
|
+
/**
|
|
69
|
+
* Atomically take up to `limit` of this crawl's pending pages to work on.
|
|
70
|
+
*
|
|
71
|
+
* Concurrent workers get disjoint sets, so a page is never fetched twice. An empty array means
|
|
72
|
+
* there is nothing left to do — that is how a worker knows to stop, not an error.
|
|
73
|
+
*
|
|
74
|
+
* Claiming starts a lease: finish the pages (mark them parsed or failed) or they are handed back
|
|
75
|
+
* to another worker once the lease expires.
|
|
76
|
+
*/
|
|
77
|
+
async claimPages(crawlJobId, limit) {
|
|
78
|
+
const result = await this.requestCollection(path('/pulse/crawl-jobs/:crawlJobId', { crawlJobId }) + '/pages/claim', CrawlJobPage.parse, { limit }, 'POST');
|
|
79
|
+
return result.getData();
|
|
80
|
+
}
|
|
67
81
|
/** @param suffix a literal trailing segment, e.g. `/pages/crawled` — not caller data. */
|
|
68
82
|
async postUrls(crawlJobId, suffix, urls) {
|
|
69
83
|
if (urls.length === 0) {
|
|
@@ -62,7 +62,7 @@ export class VendorProductEndpoint extends Endpoint {
|
|
|
62
62
|
price: product.price,
|
|
63
63
|
originalPrice: product.originalPrice,
|
|
64
64
|
shippingCost: product.shippingCost,
|
|
65
|
-
|
|
65
|
+
stockStatus: product.stockStatus,
|
|
66
66
|
properties: product.properties,
|
|
67
67
|
};
|
|
68
68
|
const result = await this.requestObject(path('/pulse/vendors/:vendorId/products', { vendorId: product.vendorId }), body, VendorProduct.parse, 'POST');
|
|
@@ -76,7 +76,7 @@ export class VendorProductEndpoint extends Endpoint {
|
|
|
76
76
|
{ op: 'add', path: 'original_price', value: product.originalPrice },
|
|
77
77
|
{ op: 'add', path: 'brand', value: product.brand },
|
|
78
78
|
{ op: 'add', path: 'shipping_cost', value: product.shippingCost },
|
|
79
|
-
{ op: 'add', path: '
|
|
79
|
+
{ op: 'add', path: 'stock_status', value: product.stockStatus },
|
|
80
80
|
{ op: 'add', path: 'url', value: product.url },
|
|
81
81
|
{ op: 'add', path: 'gtin', value: product.gtin },
|
|
82
82
|
{ op: 'add', path: 'image', value: product.image },
|
package/dist/index.d.ts
CHANGED
|
@@ -82,8 +82,10 @@ export { ResultError } from './Model/Result/ResultError.js';
|
|
|
82
82
|
export { StorageInformation } from './Model/Storage/StorageInformation.js';
|
|
83
83
|
export { StorageType } from './Model/Storage/StorageType.js';
|
|
84
84
|
export { VendorProduct } from './MarketPulse/Model/VendorProduct.js';
|
|
85
|
+
export { StockStatus, parseStockStatus, stockStatusFromBoolean } from './MarketPulse/Model/StockStatus.js';
|
|
85
86
|
export { VendorProductEndpoint } from './MarketPulse/Service/VendorProductEndpoint.js';
|
|
86
87
|
export { CrawlJob, CrawlJobStatus } from './MarketPulse/Model/CrawlJob.js';
|
|
88
|
+
export { CrawlJobPage, CrawlJobPageStatus } from './MarketPulse/Model/CrawlJobPage.js';
|
|
87
89
|
export { CrawlJobEndpoint } from './MarketPulse/Service/CrawlJobEndpoint.js';
|
|
88
90
|
export { VendorProductPriceHistoryEntry } from './MarketPulse/Model/VendorProductPriceHistoryEntry.js';
|
|
89
91
|
export { Catalog } from './Catalog/Model/Catalog.js';
|
package/dist/index.js
CHANGED
|
@@ -68,8 +68,10 @@ export { ResultError } from './Model/Result/ResultError.js';
|
|
|
68
68
|
export { StorageInformation } from './Model/Storage/StorageInformation.js';
|
|
69
69
|
export { StorageType } from './Model/Storage/StorageType.js';
|
|
70
70
|
export { VendorProduct } from './MarketPulse/Model/VendorProduct.js';
|
|
71
|
+
export { parseStockStatus, stockStatusFromBoolean } from './MarketPulse/Model/StockStatus.js';
|
|
71
72
|
export { VendorProductEndpoint } from './MarketPulse/Service/VendorProductEndpoint.js';
|
|
72
73
|
export { CrawlJob } from './MarketPulse/Model/CrawlJob.js';
|
|
74
|
+
export { CrawlJobPage } from './MarketPulse/Model/CrawlJobPage.js';
|
|
73
75
|
export { CrawlJobEndpoint } from './MarketPulse/Service/CrawlJobEndpoint.js';
|
|
74
76
|
export { VendorProductPriceHistoryEntry } from './MarketPulse/Model/VendorProductPriceHistoryEntry.js';
|
|
75
77
|
export { Catalog } from './Catalog/Model/Catalog.js';
|
package/dist/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const VERSION = "1.
|
|
1
|
+
export declare const VERSION = "1.105.0";
|
package/dist/version.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const VERSION = "1.
|
|
1
|
+
export const VERSION = "1.105.0";
|