@propertysimple/client 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/API.md +79 -0
- package/README.md +348 -0
- package/client/src/directory-sync.d.ts +123 -0
- package/client/src/directory-sync.js +404 -0
- package/client/src/index.d.ts +5123 -0
- package/client/src/index.js +285 -0
- package/client/src/managed-sync.d.ts +129 -0
- package/client/src/managed-sync.js +410 -0
- package/contracts/src/catalog-derived.d.ts +7 -0
- package/contracts/src/catalog-derived.js +78 -0
- package/contracts/src/catalog-fields.d.ts +231 -0
- package/contracts/src/catalog-fields.js +194 -0
- package/contracts/src/catalog-inventory.d.ts +14 -0
- package/contracts/src/catalog-inventory.js +10 -0
- package/contracts/src/directory-execution.d.ts +320 -0
- package/contracts/src/directory-execution.js +122 -0
- package/contracts/src/directory-identity.d.ts +1255 -0
- package/contracts/src/directory-identity.js +129 -0
- package/contracts/src/directory-sales.d.ts +333 -0
- package/contracts/src/directory-sales.js +102 -0
- package/contracts/src/directory.d.ts +1088 -0
- package/contracts/src/directory.js +179 -0
- package/contracts/src/index.d.ts +18 -0
- package/contracts/src/index.js +18 -0
- package/contracts/src/intelligence.d.ts +366 -0
- package/contracts/src/intelligence.js +157 -0
- package/contracts/src/location-serving.d.ts +13 -0
- package/contracts/src/location-serving.js +89 -0
- package/contracts/src/location.d.ts +73 -0
- package/contracts/src/location.js +75 -0
- package/contracts/src/openapi.d.ts +22 -0
- package/contracts/src/openapi.js +109 -0
- package/contracts/src/operations.d.ts +8316 -0
- package/contracts/src/operations.js +56 -0
- package/contracts/src/opportunity-events.d.ts +38 -0
- package/contracts/src/opportunity-events.js +69 -0
- package/contracts/src/platform-api.d.ts +12 -0
- package/contracts/src/platform-api.js +1 -0
- package/contracts/src/professional-intelligence.d.ts +256 -0
- package/contracts/src/professional-intelligence.js +90 -0
- package/contracts/src/property-features.d.ts +94 -0
- package/contracts/src/property-features.js +398 -0
- package/contracts/src/publication.d.ts +44 -0
- package/contracts/src/publication.js +42 -0
- package/contracts/src/routes.d.ts +437 -0
- package/contracts/src/routes.js +496 -0
- package/contracts/src/schemas.d.ts +3198 -0
- package/contracts/src/schemas.js +674 -0
- package/contracts/src/search.d.ts +2630 -0
- package/contracts/src/search.js +223 -0
- package/contracts/src/smart-feed.d.ts +3731 -0
- package/contracts/src/smart-feed.js +109 -0
- package/openapi.json +43542 -0
- package/package.json +60 -0
|
@@ -0,0 +1,404 @@
|
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
|
+
import { operations, SmartFeedSelectorSchema, SmartFeedResourceSchema, } from '../../contracts/src/index.js';
|
|
3
|
+
import { ApiError } from './index.js';
|
|
4
|
+
import { freshLocationProjection } from '../../contracts/src/location-serving.js';
|
|
5
|
+
import { ProjectionUnavailableError, } from './managed-sync.js';
|
|
6
|
+
const newCheckpoint = () => ({
|
|
7
|
+
phase: 'create',
|
|
8
|
+
generation: randomUUID(),
|
|
9
|
+
lastSequence: '0',
|
|
10
|
+
visibleSequence: '0',
|
|
11
|
+
});
|
|
12
|
+
const seq = (value) => {
|
|
13
|
+
if (!/^\d+$/.test(value))
|
|
14
|
+
throw new Error('Invalid smart-feed sequence.');
|
|
15
|
+
return BigInt(value);
|
|
16
|
+
};
|
|
17
|
+
const key = (resource) => `${resource.resource_type}:${resource.resource_id}`;
|
|
18
|
+
const afterKey = (value = '') => {
|
|
19
|
+
if (!value)
|
|
20
|
+
return ['', ''];
|
|
21
|
+
const delimiter = value.indexOf(':');
|
|
22
|
+
const type = value.slice(0, delimiter), id = value.slice(delimiter + 1);
|
|
23
|
+
if (!['entity', 'listing', 'relationship', 'identity_resolution'].includes(type) ||
|
|
24
|
+
!id ||
|
|
25
|
+
id.length > 256)
|
|
26
|
+
throw new Error('Use a typed resource key for pagination.');
|
|
27
|
+
return [type, id];
|
|
28
|
+
};
|
|
29
|
+
const serveResource = (payload, at) => {
|
|
30
|
+
const resource = SmartFeedResourceSchema.parse(payload);
|
|
31
|
+
return resource.resource_type === 'listing'
|
|
32
|
+
? { ...resource, payload: freshLocationProjection(resource.payload, at) }
|
|
33
|
+
: resource;
|
|
34
|
+
};
|
|
35
|
+
/**
|
|
36
|
+
* A consumer-owned typed projection. Versions received for an incomplete group are durable but
|
|
37
|
+
* invisible. Completing a group changes one visibility pointer rather than applying an unbounded
|
|
38
|
+
* mutation transaction. Product-owned users, leads, slugs and billing never enter this schema.
|
|
39
|
+
*/
|
|
40
|
+
export class PostgresDirectoryTarget {
|
|
41
|
+
db;
|
|
42
|
+
configuration;
|
|
43
|
+
selector;
|
|
44
|
+
fingerprint;
|
|
45
|
+
constructor(db, configuration) {
|
|
46
|
+
this.db = db;
|
|
47
|
+
this.configuration = configuration;
|
|
48
|
+
for (const value of [
|
|
49
|
+
configuration.streamId,
|
|
50
|
+
configuration.datasetId,
|
|
51
|
+
configuration.sourceNamespace,
|
|
52
|
+
])
|
|
53
|
+
if (!value || value.length > 256)
|
|
54
|
+
throw new Error('Sync identities must contain 1–256 characters.');
|
|
55
|
+
this.selector = SmartFeedSelectorSchema.parse(configuration.selector ?? {});
|
|
56
|
+
if (this.selector.party_id && this.selector.mode !== 'combined')
|
|
57
|
+
throw new Error('A party selector requires combined mode.');
|
|
58
|
+
this.fingerprint = createHash('sha256')
|
|
59
|
+
.update(JSON.stringify([configuration.datasetId, configuration.sourceNamespace, this.selector]))
|
|
60
|
+
.digest('hex');
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Explicit approved install/upgrade. v2 keeps resource_type as text, adds version metadata,
|
|
64
|
+
* and invalidates every v1 checkpoint so identity bindings enter through a complete snapshot.
|
|
65
|
+
* Stable local UUID mappings and old versions remain for product-owned recovery/pruning.
|
|
66
|
+
* Quiesce old SDK workers before invoking this; initialization is not a request-time action.
|
|
67
|
+
*/
|
|
68
|
+
async initialize() {
|
|
69
|
+
await this.db.transaction(async (tx) => {
|
|
70
|
+
await tx.query(`create schema if not exists consumer_smart_feed;
|
|
71
|
+
create table if not exists consumer_smart_feed.schema_version(singleton integer primary key check(singleton=1),version integer not null);
|
|
72
|
+
insert into consumer_smart_feed.schema_version(singleton,version) values(1,1) on conflict do nothing;
|
|
73
|
+
alter table consumer_smart_feed.schema_version enable row level security;
|
|
74
|
+
revoke all on consumer_smart_feed.schema_version from public;`);
|
|
75
|
+
const version = (await tx.query(`select version from consumer_smart_feed.schema_version where singleton=1 for update`)).rows[0].version;
|
|
76
|
+
if (![1, 2].includes(version))
|
|
77
|
+
throw new Error('Unsupported consumer smart-feed schema version; use a compatible SDK.');
|
|
78
|
+
await tx.query(`
|
|
79
|
+
create table if not exists consumer_smart_feed.streams(id text primary key,fingerprint text not null,version bigint not null default 0,checkpoint jsonb not null,active_generation text,visible_seq bigint not null default 0,synchronized_at timestamptz);
|
|
80
|
+
create table if not exists consumer_smart_feed.identities(stream_id text not null references consumer_smart_feed.streams(id),resource_type text not null,resource_id text not null,local_id uuid not null,primary key(stream_id,resource_type,resource_id),unique(stream_id,local_id));
|
|
81
|
+
create table if not exists consumer_smart_feed.versions(stream_id text not null references consumer_smart_feed.streams(id),generation text not null,resource_type text not null,resource_id text not null,sequence bigint not null,payload jsonb,primary key(stream_id,generation,resource_type,resource_id,sequence));
|
|
82
|
+
create index if not exists smart_feed_projection_read on consumer_smart_feed.versions(stream_id,generation,resource_type,resource_id,sequence desc);
|
|
83
|
+
create index if not exists smart_feed_projection_changes on consumer_smart_feed.versions(stream_id,generation,sequence);
|
|
84
|
+
create table if not exists consumer_smart_feed.invalidations(id uuid primary key,stream_id text not null references consumer_smart_feed.streams(id),generation text not null,kind text not null check(kind in('reset','group')),from_sequence bigint not null,through_sequence bigint not null,created_at timestamptz not null default now(),acknowledged_at timestamptz,lease_token uuid,lease_until timestamptz);
|
|
85
|
+
create index if not exists smart_feed_invalidations_pending on consumer_smart_feed.invalidations(stream_id,created_at) where acknowledged_at is null;
|
|
86
|
+
alter table consumer_smart_feed.streams enable row level security;
|
|
87
|
+
alter table consumer_smart_feed.identities enable row level security;
|
|
88
|
+
alter table consumer_smart_feed.versions enable row level security;
|
|
89
|
+
alter table consumer_smart_feed.invalidations enable row level security;
|
|
90
|
+
revoke all on schema consumer_smart_feed from public;
|
|
91
|
+
revoke all on consumer_smart_feed.streams,consumer_smart_feed.identities,consumer_smart_feed.versions,consumer_smart_feed.invalidations from public;`);
|
|
92
|
+
if (version === 1) {
|
|
93
|
+
// A single schema transaction fences old page writers through stream versions,
|
|
94
|
+
// hides stale projections immediately, and emits durable product invalidations.
|
|
95
|
+
await tx.query(`with reset as(update consumer_smart_feed.streams set version=version+1,active_generation=null,visible_seq=0,synchronized_at=null,checkpoint=jsonb_build_object('phase','create','generation',gen_random_uuid()::text,'lastSequence','0','visibleSequence','0') returning id,checkpoint)
|
|
96
|
+
insert into consumer_smart_feed.invalidations(id,stream_id,generation,kind,from_sequence,through_sequence) select gen_random_uuid(),id,checkpoint->>'generation','reset',0,0 from reset`);
|
|
97
|
+
await tx.query(`update consumer_smart_feed.schema_version set version=2 where singleton=1`);
|
|
98
|
+
}
|
|
99
|
+
await tx.query(`insert into consumer_smart_feed.streams(id,fingerprint,checkpoint) values($1,$2,$3::jsonb) on conflict(id) do nothing`, [this.configuration.streamId, this.fingerprint, JSON.stringify(newCheckpoint())]);
|
|
100
|
+
});
|
|
101
|
+
await this.state();
|
|
102
|
+
}
|
|
103
|
+
async state() {
|
|
104
|
+
const row = (await this.db.query(`select *,version::text,visible_seq::text from consumer_smart_feed.streams where id=$1`, [this.configuration.streamId])).rows[0];
|
|
105
|
+
if (!row)
|
|
106
|
+
throw new Error('Initialize the directory target first.');
|
|
107
|
+
if (row.fingerprint !== this.fingerprint)
|
|
108
|
+
throw new Error('Feed scope/selector changed; create a new stream.');
|
|
109
|
+
return row;
|
|
110
|
+
}
|
|
111
|
+
async commit(expected, next, write, options = {}) {
|
|
112
|
+
return this.db.transaction(async (tx) => {
|
|
113
|
+
const current = (await tx.query(`select version::text from consumer_smart_feed.streams where id=$1 for update`, [this.configuration.streamId])).rows[0];
|
|
114
|
+
if (current.version !== expected.version)
|
|
115
|
+
return false;
|
|
116
|
+
await write(tx);
|
|
117
|
+
await tx.query(`update consumer_smart_feed.streams set checkpoint=$2::jsonb,version=version+1,visible_seq=$3,active_generation=case when $4 then $5 else active_generation end,synchronized_at=case when $6 then now() else synchronized_at end where id=$1`, [
|
|
118
|
+
this.configuration.streamId,
|
|
119
|
+
JSON.stringify(next),
|
|
120
|
+
next.visibleSequence,
|
|
121
|
+
options.activate ?? false,
|
|
122
|
+
next.generation,
|
|
123
|
+
options.synchronized ?? false,
|
|
124
|
+
]);
|
|
125
|
+
return true;
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
/** Two set-based writes per page, independent of page size. Checkpoint commits in the caller. */
|
|
129
|
+
async writePage(tx, generation, items) {
|
|
130
|
+
if (!items.length)
|
|
131
|
+
return;
|
|
132
|
+
const records = items.map((row) => ({ ...row, local_id: randomUUID() }));
|
|
133
|
+
await tx.query(`insert into consumer_smart_feed.identities(stream_id,resource_type,resource_id,local_id) select $1,resource_type,resource_id,local_id from jsonb_to_recordset($2::jsonb) x(resource_type text,resource_id text,local_id uuid) on conflict(stream_id,resource_type,resource_id) do nothing`, [this.configuration.streamId, JSON.stringify(records)]);
|
|
134
|
+
await tx.query(`insert into consumer_smart_feed.versions(stream_id,generation,resource_type,resource_id,sequence,payload) select $1,$2,resource_type,resource_id,sequence,nullif(resource,'null'::jsonb) from jsonb_to_recordset($3::jsonb) x(resource_type text,resource_id text,sequence bigint,resource jsonb)`, [this.configuration.streamId, generation, JSON.stringify(records)]);
|
|
135
|
+
}
|
|
136
|
+
async invalidate(tx, generation, kind, from, through) {
|
|
137
|
+
await tx.query(`insert into consumer_smart_feed.invalidations(id,stream_id,generation,kind,from_sequence,through_sequence) values($1,$2,$3,$4,$5,$6)`, [randomUUID(), this.configuration.streamId, generation, kind, from, through]);
|
|
138
|
+
}
|
|
139
|
+
async reset(expected, blocked = false) {
|
|
140
|
+
const next = newCheckpoint();
|
|
141
|
+
if (blocked)
|
|
142
|
+
next.phase = 'blocked';
|
|
143
|
+
return this.commit(expected, next, async (tx) => {
|
|
144
|
+
await tx.query(`update consumer_smart_feed.streams set active_generation=null,synchronized_at=null where id=$1`, [this.configuration.streamId]);
|
|
145
|
+
await this.invalidate(tx, next.generation, 'reset', '0', '0');
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
async resume() {
|
|
149
|
+
const state = await this.state();
|
|
150
|
+
if (state.checkpoint.phase !== 'blocked')
|
|
151
|
+
throw new Error('Only blocked streams require resume.');
|
|
152
|
+
await this.reset(state);
|
|
153
|
+
}
|
|
154
|
+
async mapExistingIdentity(resourceType, resourceId, localId) {
|
|
155
|
+
await this.db.transaction(async (tx) => {
|
|
156
|
+
await tx.query(`insert into consumer_smart_feed.identities(stream_id,resource_type,resource_id,local_id) values($1,$2,$3,$4) on conflict(stream_id,resource_type,resource_id) do nothing`, [this.configuration.streamId, resourceType, resourceId, localId]);
|
|
157
|
+
const row = (await tx.query(`select local_id from consumer_smart_feed.identities where stream_id=$1 and resource_type=$2 and resource_id=$3`, [this.configuration.streamId, resourceType, resourceId])).rows[0];
|
|
158
|
+
if (row.local_id !== localId.toLowerCase())
|
|
159
|
+
throw new Error('Existing product identities cannot be remapped.');
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
async readable(tx, options) {
|
|
163
|
+
if (!Number.isFinite(options.maxStalenessMs) || options.maxStalenessMs <= 0)
|
|
164
|
+
throw new Error('An explicit positive freshness budget is required.');
|
|
165
|
+
const row = (await tx.query(`select active_generation is not null and synchronized_at>=now()-($2::double precision*interval '1 millisecond') as ready,checkpoint,now()::text as now_at from consumer_smart_feed.streams where id=$1 for share`, [this.configuration.streamId, options.maxStalenessMs])).rows[0];
|
|
166
|
+
if (!row?.ready)
|
|
167
|
+
throw new ProjectionUnavailableError();
|
|
168
|
+
for (const [budget, field] of [
|
|
169
|
+
[options.maxSourceStalenessMs, 'last_completed_enumeration_at'],
|
|
170
|
+
[options.maxFullEnumerationStalenessMs, 'last_full_enumeration_at'],
|
|
171
|
+
]) {
|
|
172
|
+
if (budget === undefined)
|
|
173
|
+
continue;
|
|
174
|
+
if (!Number.isFinite(budget) || budget <= 0)
|
|
175
|
+
throw new Error('Source freshness budgets must be positive.');
|
|
176
|
+
if (!row.checkpoint.sourceFreshness?.length ||
|
|
177
|
+
row.checkpoint.sourceFreshness.some((source) => {
|
|
178
|
+
const at = source[field] ? Date.parse(source[field]) : NaN;
|
|
179
|
+
const now = Date.parse(row.now_at);
|
|
180
|
+
return !Number.isFinite(at) || now - at > budget || at > now + 60_000;
|
|
181
|
+
}))
|
|
182
|
+
throw new ProjectionUnavailableError('Source evidence is outside its freshness budget.');
|
|
183
|
+
}
|
|
184
|
+
return new Date(row.now_at);
|
|
185
|
+
}
|
|
186
|
+
async list(options) {
|
|
187
|
+
const limit = options.limit ?? 100;
|
|
188
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 200)
|
|
189
|
+
throw new Error('Read limit must be 1–200.');
|
|
190
|
+
const [afterType, afterId] = afterKey(options.after);
|
|
191
|
+
return this.db.transaction(async (tx) => {
|
|
192
|
+
const at = await this.readable(tx, options);
|
|
193
|
+
const rows = (await tx.query(`with latest as(select distinct on(v.resource_type,v.resource_id) v.resource_type,v.resource_id,v.payload from consumer_smart_feed.versions v join consumer_smart_feed.streams s on s.id=v.stream_id and s.active_generation=v.generation where s.id=$1 and v.sequence<=s.visible_seq and ($2::text is null or v.resource_type=$2) and (v.resource_type,v.resource_id)>($3,$4) order by v.resource_type,v.resource_id,v.sequence desc) select l.payload,i.local_id from latest l join consumer_smart_feed.identities i on i.stream_id=$1 and i.resource_type=l.resource_type and i.resource_id=l.resource_id where l.payload is not null order by l.resource_type,l.resource_id limit $5`, [this.configuration.streamId, options.resourceType ?? null, afterType, afterId, limit])).rows;
|
|
194
|
+
return rows.map((row) => ({
|
|
195
|
+
resource: serveResource(row.payload, at),
|
|
196
|
+
local_id: row.local_id,
|
|
197
|
+
}));
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
async get(resourceType, resourceId, options) {
|
|
201
|
+
return this.db.transaction(async (tx) => {
|
|
202
|
+
const at = await this.readable(tx, options);
|
|
203
|
+
const row = (await tx.query(`select v.payload from consumer_smart_feed.versions v join consumer_smart_feed.streams s on s.id=v.stream_id and s.active_generation=v.generation where s.id=$1 and v.resource_type=$2 and v.resource_id=$3 and v.sequence<=s.visible_seq order by v.sequence desc limit 1`, [this.configuration.streamId, resourceType, resourceId])).rows[0];
|
|
204
|
+
return row?.payload ? serveResource(row.payload, at) : null;
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
async claimInvalidations(limit = 100, leaseSeconds = 60) {
|
|
208
|
+
if (!Number.isInteger(limit) ||
|
|
209
|
+
limit < 1 ||
|
|
210
|
+
limit > 200 ||
|
|
211
|
+
!Number.isInteger(leaseSeconds) ||
|
|
212
|
+
leaseSeconds < 1 ||
|
|
213
|
+
leaseSeconds > 3600)
|
|
214
|
+
throw new Error('Invalid invalidation lease bounds.');
|
|
215
|
+
return (await this.db.query(`with selected as(select id from consumer_smart_feed.invalidations where stream_id=$1 and acknowledged_at is null and (lease_until is null or lease_until<now()) order by created_at,id limit $2 for update skip locked) update consumer_smart_feed.invalidations o set lease_token=$3,lease_until=now()+($4::integer*interval '1 second') from selected where o.id=selected.id returning o.id,o.kind,o.generation,o.from_sequence::text,o.through_sequence::text,o.lease_token`, [this.configuration.streamId, limit, randomUUID(), leaseSeconds])).rows;
|
|
216
|
+
}
|
|
217
|
+
/** Page every affected key before acknowledging a group; resolve current state for cache effects. */
|
|
218
|
+
async invalidationItems(event, after = '', limit = 100) {
|
|
219
|
+
if (event.kind === 'reset')
|
|
220
|
+
throw new Error('Reset invalidations require a complete product cache/index rebuild.');
|
|
221
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 200)
|
|
222
|
+
throw new Error('Invalid page size.');
|
|
223
|
+
const [afterType, afterId] = afterKey(after);
|
|
224
|
+
return (await this.db.query(`select distinct v.resource_type,v.resource_id from consumer_smart_feed.versions v join consumer_smart_feed.invalidations o on o.stream_id=v.stream_id and o.generation=v.generation where o.id=$1 and o.stream_id=$2 and o.lease_token=$3 and o.lease_until>=now() and o.acknowledged_at is null and v.sequence>=o.from_sequence and v.sequence<=o.through_sequence and(v.resource_type,v.resource_id)>($4,$5) order by v.resource_type,v.resource_id limit $6`, [event.id, this.configuration.streamId, event.lease_token, afterType, afterId, limit])).rows;
|
|
225
|
+
}
|
|
226
|
+
async acknowledgeInvalidation(event) {
|
|
227
|
+
return ((await this.db.query(`update consumer_smart_feed.invalidations set acknowledged_at=now() where id=$1 and stream_id=$2 and lease_token=$3 and lease_until>=now() and acknowledged_at is null`, [event.id, this.configuration.streamId, event.lease_token])).rowCount === 1);
|
|
228
|
+
}
|
|
229
|
+
/** Preserve staged groups and unacknowledged invalidations while reclaiming bounded history. */
|
|
230
|
+
async pruneObsolete(limit = 1000) {
|
|
231
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 10000)
|
|
232
|
+
throw new Error('Invalid pruning bound.');
|
|
233
|
+
return this.db.transaction(async (tx) => {
|
|
234
|
+
await tx.query(`select id from consumer_smart_feed.streams where id=$1 for update`, [
|
|
235
|
+
this.configuration.streamId,
|
|
236
|
+
]);
|
|
237
|
+
return (await tx.query(`delete from consumer_smart_feed.versions where(stream_id,generation,resource_type,resource_id,sequence) in(
|
|
238
|
+
select v.stream_id,v.generation,v.resource_type,v.resource_id,v.sequence from consumer_smart_feed.versions v join consumer_smart_feed.streams s on s.id=v.stream_id
|
|
239
|
+
where s.id=$1 and not exists(select 1 from consumer_smart_feed.invalidations o where o.stream_id=v.stream_id and o.generation=v.generation and o.acknowledged_at is null and(o.kind='reset' or v.sequence between o.from_sequence and o.through_sequence))
|
|
240
|
+
and((v.generation is distinct from s.active_generation and v.generation<>s.checkpoint->>'generation') or(v.generation=s.active_generation and v.sequence<=s.visible_seq and exists(select 1 from consumer_smart_feed.versions n where n.stream_id=v.stream_id and n.generation=v.generation and n.resource_type=v.resource_type and n.resource_id=v.resource_id and n.sequence>v.sequence and n.sequence<=s.visible_seq))) limit $2)`, [this.configuration.streamId, limit])).rowCount;
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
export class DirectorySyncWorker {
|
|
245
|
+
source;
|
|
246
|
+
target;
|
|
247
|
+
pageSize;
|
|
248
|
+
constructor(source, target, pageSize = 100) {
|
|
249
|
+
this.source = source;
|
|
250
|
+
this.target = target;
|
|
251
|
+
this.pageSize = pageSize;
|
|
252
|
+
if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > 200)
|
|
253
|
+
throw new Error('Feed page size must be 1–200.');
|
|
254
|
+
}
|
|
255
|
+
async step(options = {}) {
|
|
256
|
+
const state = await this.target.state(), checkpoint = state.checkpoint;
|
|
257
|
+
const result = (status, applied = 0) => ({ status, applied });
|
|
258
|
+
if (checkpoint.phase === 'blocked')
|
|
259
|
+
return result('blocked');
|
|
260
|
+
try {
|
|
261
|
+
if (checkpoint.phase === 'create') {
|
|
262
|
+
const snapshot = operations.createDirectorySnapshot.output.parse(await this.source.directory.createSnapshot({ dataset_id: this.target.configuration.datasetId, ...this.target.selector }, { ...options, idempotencyKey: `smart-feed-snapshot-${checkpoint.generation}` }));
|
|
263
|
+
if (snapshot.dataset_id !== this.target.configuration.datasetId ||
|
|
264
|
+
JSON.stringify(snapshot.selector) !== JSON.stringify(this.target.selector))
|
|
265
|
+
throw new Error('Feed snapshot scope differs from the requested stream.');
|
|
266
|
+
if (snapshot.status === 'expired')
|
|
267
|
+
return result((await this.target.reset(state)) ? 'reset' : 'conflict');
|
|
268
|
+
seq(snapshot.snapshot_sequence);
|
|
269
|
+
const next = {
|
|
270
|
+
...checkpoint,
|
|
271
|
+
phase: 'snapshot',
|
|
272
|
+
snapshotId: snapshot.id,
|
|
273
|
+
snapshotSequence: snapshot.snapshot_sequence,
|
|
274
|
+
lastSequence: snapshot.snapshot_sequence,
|
|
275
|
+
visibleSequence: snapshot.snapshot_sequence,
|
|
276
|
+
replayCursor: snapshot.replay_cursor,
|
|
277
|
+
};
|
|
278
|
+
return result((await this.target.commit(state, next, async () => { })) ? 'progress' : 'conflict');
|
|
279
|
+
}
|
|
280
|
+
if (checkpoint.phase === 'snapshot') {
|
|
281
|
+
const page = operations.listDirectorySnapshotItems.output.parse(await this.source.directory.snapshotItems(checkpoint.snapshotId, {
|
|
282
|
+
limit: this.pageSize,
|
|
283
|
+
...(checkpoint.snapshotCursor ? { cursor: checkpoint.snapshotCursor } : {}),
|
|
284
|
+
}, options));
|
|
285
|
+
if (page.data.length > this.pageSize ||
|
|
286
|
+
page.meta.snapshot_sequence !== checkpoint.snapshotSequence ||
|
|
287
|
+
new Set(page.data.map(key)).size !== page.data.length ||
|
|
288
|
+
(page.has_more &&
|
|
289
|
+
(!page.data.length ||
|
|
290
|
+
!page.next_cursor ||
|
|
291
|
+
page.next_cursor === checkpoint.snapshotCursor)))
|
|
292
|
+
throw new Error('Invalid feed snapshot pagination.');
|
|
293
|
+
const next = {
|
|
294
|
+
...checkpoint,
|
|
295
|
+
phase: page.has_more ? 'snapshot' : 'replay',
|
|
296
|
+
...(page.has_more ? { snapshotCursor: page.next_cursor } : {}),
|
|
297
|
+
sourceFreshness: page.meta.source_freshness,
|
|
298
|
+
};
|
|
299
|
+
const saved = await this.target.commit(state, next, (tx) => this.target.writePage(tx, checkpoint.generation, page.data.map((resource) => ({
|
|
300
|
+
resource_type: resource.resource_type,
|
|
301
|
+
resource_id: resource.resource_id,
|
|
302
|
+
sequence: checkpoint.snapshotSequence,
|
|
303
|
+
resource,
|
|
304
|
+
}))));
|
|
305
|
+
return result(saved ? 'progress' : 'conflict', saved ? page.data.length : 0);
|
|
306
|
+
}
|
|
307
|
+
const page = operations.listDirectoryChanges.output.parse(await this.source.directory.changes({
|
|
308
|
+
dataset_id: this.target.configuration.datasetId,
|
|
309
|
+
...this.target.selector,
|
|
310
|
+
cursor: checkpoint.replayCursor,
|
|
311
|
+
limit: this.pageSize,
|
|
312
|
+
}, options));
|
|
313
|
+
if (page.data.length > this.pageSize ||
|
|
314
|
+
(page.has_more && !page.data.length) ||
|
|
315
|
+
(page.data.length && page.next_cursor === checkpoint.replayCursor))
|
|
316
|
+
throw new Error('Invalid feed replay pagination.');
|
|
317
|
+
const next = structuredClone(checkpoint), completed = [];
|
|
318
|
+
for (const change of page.data) {
|
|
319
|
+
this.validateChange(change, next);
|
|
320
|
+
if (!next.group)
|
|
321
|
+
next.group = {
|
|
322
|
+
id: change.group_id,
|
|
323
|
+
end: change.group_end_sequence,
|
|
324
|
+
size: change.group_size,
|
|
325
|
+
next: 0,
|
|
326
|
+
start: change.sequence,
|
|
327
|
+
};
|
|
328
|
+
const group = next.group;
|
|
329
|
+
if (group.id !== change.group_id ||
|
|
330
|
+
group.end !== change.group_end_sequence ||
|
|
331
|
+
group.size !== change.group_size ||
|
|
332
|
+
group.next !== change.group_index)
|
|
333
|
+
throw new Error('Incomplete, reordered or changed feed commit group.');
|
|
334
|
+
group.next++;
|
|
335
|
+
next.lastSequence = change.sequence;
|
|
336
|
+
if (group.next === group.size) {
|
|
337
|
+
next.visibleSequence = group.end;
|
|
338
|
+
completed.push({ from: group.start, through: group.end });
|
|
339
|
+
delete next.group;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
if (!page.has_more && next.group)
|
|
343
|
+
throw new Error('Feed ended before its commit group completed.');
|
|
344
|
+
if (seq(page.meta.published_sequence) < seq(next.lastSequence))
|
|
345
|
+
throw new Error('Feed watermark is behind its changes.');
|
|
346
|
+
next.replayCursor = page.next_cursor;
|
|
347
|
+
next.sourceFreshness = page.meta.source_freshness;
|
|
348
|
+
if (!page.has_more)
|
|
349
|
+
next.visibleSequence = page.meta.published_sequence;
|
|
350
|
+
const activate = !page.has_more && state.active_generation !== checkpoint.generation;
|
|
351
|
+
const saved = await this.target.commit(state, next, async (tx) => {
|
|
352
|
+
await this.target.writePage(tx, checkpoint.generation, page.data.map((change) => ({
|
|
353
|
+
resource_type: change.resource_type,
|
|
354
|
+
resource_id: change.resource_id,
|
|
355
|
+
sequence: change.sequence,
|
|
356
|
+
resource: change.resource,
|
|
357
|
+
})));
|
|
358
|
+
if (activate)
|
|
359
|
+
await this.target.invalidate(tx, checkpoint.generation, 'reset', '0', next.visibleSequence);
|
|
360
|
+
else if (state.active_generation === checkpoint.generation)
|
|
361
|
+
for (const group of completed)
|
|
362
|
+
await this.target.invalidate(tx, checkpoint.generation, 'group', group.from, group.through);
|
|
363
|
+
}, { activate, synchronized: !page.has_more });
|
|
364
|
+
return result(saved ? (page.data.length || page.has_more ? 'progress' : 'idle') : 'conflict', saved ? page.data.length : 0);
|
|
365
|
+
}
|
|
366
|
+
catch (error) {
|
|
367
|
+
if (error instanceof ApiError && error.status === 410)
|
|
368
|
+
return result((await this.target.reset(state)) ? 'reset' : 'conflict');
|
|
369
|
+
if (error instanceof ApiError && [401, 403].includes(error.status))
|
|
370
|
+
await this.target.reset(state, true);
|
|
371
|
+
throw error;
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
validateChange(change, checkpoint) {
|
|
375
|
+
if (seq(change.sequence) <= seq(checkpoint.lastSequence) ||
|
|
376
|
+
seq(change.group_end_sequence) < seq(change.sequence) ||
|
|
377
|
+
change.group_index >= change.group_size)
|
|
378
|
+
throw new Error('Invalid feed sequence/group boundaries.');
|
|
379
|
+
if (!checkpoint.group && change.group_index !== 0)
|
|
380
|
+
throw new Error('Feed commit group starts in the middle.');
|
|
381
|
+
if (!checkpoint.group && seq(change.sequence) <= seq(checkpoint.visibleSequence))
|
|
382
|
+
throw new Error('Feed commit groups overlap.');
|
|
383
|
+
if (change.operation === 'remove' && change.resource !== null)
|
|
384
|
+
throw new Error('A removal cannot carry a resource.');
|
|
385
|
+
if (change.operation === 'upsert' &&
|
|
386
|
+
(!change.resource ||
|
|
387
|
+
change.resource.resource_id !== change.resource_id ||
|
|
388
|
+
change.resource.resource_type !== change.resource_type ||
|
|
389
|
+
change.resource.revision !== change.revision))
|
|
390
|
+
throw new Error('Feed resource identity/revision mismatch.');
|
|
391
|
+
}
|
|
392
|
+
async run(options) {
|
|
393
|
+
if (!Number.isInteger(options.maxSteps) || options.maxSteps < 1 || options.maxSteps > 10000)
|
|
394
|
+
throw new Error('A bounded maxSteps is required.');
|
|
395
|
+
const results = [];
|
|
396
|
+
for (let count = 0; count < options.maxSteps; count++) {
|
|
397
|
+
const result = await this.step(options);
|
|
398
|
+
results.push(result);
|
|
399
|
+
if (['idle', 'blocked', 'conflict'].includes(result.status))
|
|
400
|
+
break;
|
|
401
|
+
}
|
|
402
|
+
return results;
|
|
403
|
+
}
|
|
404
|
+
}
|