@flowselections/floriday-authenticatie-module 1.0.17 → 1.0.18

package/dist-lib/lib/floricode-sync.server.js

@@ -0,0 +1,211 @@
+// ============================================================================
+// Floricode generic incremental sync engine.
+// ----------------------------------------------------------------------------
+// One resource registry. Each entry describes:
+//   - REST endpoint on api.floricode.com
+//   - OAuth scope to request
+//   - unique-key columns (used both for `record_id` derivation and $orderby)
+// The engine reads a per-resource cursor from `floricode_sync_state`:
+//   - `last_change_date_time` is the incremental watermark
+//   - `cursor_skip` handles paging WITHIN a single watermark run
+// When a run finishes cleanly, watermark advances and cursor resets.
+// Rows land in `public.floricode_records` with (resource, record_id) as key.
+// ============================================================================
+import { supabaseAdmin } from "../integrations/supabase/client.server";
+import { getFloricodeAccessToken } from "./floricode-credentials.server";
+export const FLORICODE_RESOURCES = [
+    // VBN
+    { resource: "FeatureType", endpoint: "/v2/VBN/FeatureType", scope: "vbn_featuretype:read", keyColumns: ["id"] },
+    { resource: "FeatureValue", endpoint: "/v2/VBN/FeatureValue", scope: "vbn_featurevalue:read", keyColumns: ["feature_type_id", "code"] },
+    { resource: "VbnProduct", endpoint: "/v2/VBN/Product", scope: "vbn_product:read", keyColumns: ["id"] },
+    { resource: "PackagingType", endpoint: "/v2/VBN/PackagingType", scope: "vbn_packagingtype:read", keyColumns: ["id"] },
+    // Plant/Packaging hebben géén `id`-veld → composite keys uit de EF-modellen.
+    { resource: "BotanicalName", endpoint: "/v2/VBN/Plant", scope: "vbn_plant:read", keyColumns: ["registrator_id", "taxonomic_number", "entry_date"] },
+    { resource: "LoadCarrier", endpoint: "/v2/VBN/Packaging", scope: "vbn_packaging:read", keyColumns: ["code"] },
+    // FEC (voorheen CAO — Floricode heeft de company-namespace verhuisd)
+    { resource: "Company", endpoint: "/v2/FEC/Company", scope: "fec_company:read", keyColumns: ["id"] },
+    { resource: "CompanyLocation", endpoint: "/v2/FEC/CompanyLocation", scope: "fec_location:read", keyColumns: ["location_id"] },
+    { resource: "CompanyRole", endpoint: "/v2/FEC/CompanyRole", scope: "fec_companyrole:read", keyColumns: ["id"] },
+    { resource: "CompanyLevel", endpoint: "/v2/FEC/CompanyLevel", scope: "fec_companylevel:read", keyColumns: ["id"] },
+    { resource: "LocationType", endpoint: "/v2/FEC/LocationType", scope: "fec_locationtype:read", keyColumns: ["id"] },
+    // ISO/FLC-resources zijn verwijderd: Floricode geeft de bijbehorende scopes
+    // (`iso_country:read`, `flc_*`) niet vrij aan onze Client ID. Weer aanzetten
+    // zodra Floricode support die scopes toekent.
+];
+const BASE_URL = "https://api.floricode.com";
+const DEFAULT_BATCH = 5000;
+function buildRecordId(def, row) {
+    return def.keyColumns.map((k) => String(row[k] ?? "")).join("|");
+}
+async function fetchPage(def, token, opts) {
+    const url = new URL(BASE_URL + def.endpoint);
+    url.searchParams.set("$top", String(opts.top));
+    url.searchParams.set("$skip", String(opts.skip));
+    url.searchParams.set("$orderby", ["change_date_time", ...def.keyColumns].join(","));
+    if (opts.sinceIso) {
+        url.searchParams.set("$filter", `change_date_time gt ${opts.sinceIso}`);
+    }
+    const res = await fetch(url.toString(), { headers: { authorization: `Bearer ${token}` } });
+    const json = await res.json().catch(() => ({}));
+    if (!res.ok) {
+        const upstream = json?.error?.message ?? json?.error ?? res.statusText;
+        if (res.status === 404) {
+            throw new Error(`Floricode ${def.resource} endpoint niet beschikbaar (404) op ${def.endpoint}. Controleer of de scope '${def.scope}' voor deze Client ID is geactiveerd in het Floricode-portaal.`);
+        }
+        if (res.status === 401 || res.status === 403) {
+            throw new Error(`Floricode ${def.resource} niet geautoriseerd (${res.status}). Scope '${def.scope}' is niet toegekend aan de huidige Client ID.`);
+        }
+        throw new Error(`Floricode ${def.resource} pagina mislukt (${res.status}): ${upstream}`);
+    }
+    return Array.isArray(json?.value) ? json.value : [];
+}
+async function upsertRows(def, rows) {
+    if (rows.length === 0)
+        return 0;
+    const now = new Date().toISOString();
+    // In-batch dedupe op composite record_id. Floricode kan binnen één pagina
+    // dubbele sleutels teruggeven; zonder dedupe crasht de upsert met
+    // "ON CONFLICT DO UPDATE command cannot affect row a second time".
+    const dedup = new Map();
+    for (const r of rows) {
+        const record_id = buildRecordId(def, r);
+        if (!record_id)
+            continue;
+        dedup.set(record_id, {
+            resource: def.resource,
+            record_id,
+            data: r,
+            change_date_time: r?.change_date_time ?? null,
+            last_synced_at: now,
+        });
+    }
+    const payload = Array.from(dedup.values());
+    // Kleine chunks (100) om Supabase Disk IO-pieken tijdens de initiële
+    // backfill te dempen; grote JSONB-upserts drukken WAL + toast zwaar.
+    const CHUNK = 100;
+    let done = 0;
+    for (let i = 0; i < payload.length; i += CHUNK) {
+        const part = payload.slice(i, i + CHUNK);
+        const { error } = await supabaseAdmin
+            .from("floricode_records")
+            .upsert(part, { onConflict: "resource,record_id" });
+        if (error)
+            throw new Error(`Upsert ${def.resource} mislukt: ${error.message}`);
+        done += part.length;
+    }
+    return done;
+}
+export async function syncFloricodeResource(resource, opts = {}) {
+    const def = FLORICODE_RESOURCES.find((r) => r.resource === resource);
+    if (!def)
+        throw new Error(`Onbekende Floricode-resource: ${resource}`);
+    const batchSize = opts.batchSize ?? DEFAULT_BATCH;
+    const maxBatches = Math.max(1, opts.maxBatches ?? 3);
+    const resume = opts.resume ?? true;
+    const startedAt = Date.now();
+    const { data: state } = await supabaseAdmin
+        .from("floricode_sync_state")
+        .select("cursor_skip, last_change_date_time")
+        .eq("resource", def.resource)
+        .maybeSingle();
+    let skip = resume ? Number(state?.cursor_skip ?? 0) : 0;
+    const sinceIso = state?.last_change_date_time ?? null;
+    await supabaseAdmin
+        .from("floricode_sync_state")
+        .upsert({
+        resource: def.resource,
+        status: "running",
+        error_message: null,
+        updated_at: new Date().toISOString(),
+    }, { onConflict: "resource" });
+    const token = await getFloricodeAccessToken(def.scope);
+    let fetched = 0;
+    let upserted = 0;
+    let batches = 0;
+    let latestChange = sinceIso;
+    let finished = false;
+    try {
+        for (let i = 0; i < maxBatches; i++) {
+            const page = await fetchPage(def, token, { skip, top: batchSize, sinceIso });
+            batches++;
+            if (page.length === 0) {
+                finished = true;
+                break;
+            }
+            fetched += page.length;
+            upserted += await upsertRows(def, page);
+            for (const row of page) {
+                const cd = row?.change_date_time;
+                if (cd && (!latestChange || cd > latestChange))
+                    latestChange = cd;
+            }
+            skip += page.length;
+            // Cursor + watermark per batch persistent maken zodat een crash of
+            // Worker-timeout hervat vanaf de laatste geslaagde pagina.
+            await supabaseAdmin
+                .from("floricode_sync_state")
+                .upsert({
+                resource: def.resource,
+                status: "running",
+                error_message: null,
+                cursor_skip: skip,
+                last_change_date_time: latestChange,
+                last_sync_at: new Date().toISOString(),
+                updated_at: new Date().toISOString(),
+            }, { onConflict: "resource" });
+            if (page.length < batchSize) {
+                finished = true;
+                break;
+            }
+        }
+        await supabaseAdmin
+            .from("floricode_sync_state")
+            .upsert({
+            resource: def.resource,
+            status: "ok",
+            error_message: null,
+            last_sync_at: new Date().toISOString(),
+            cursor_skip: finished ? 0 : skip,
+            last_change_date_time: finished ? latestChange : sinceIso,
+            updated_at: new Date().toISOString(),
+        }, { onConflict: "resource" });
+    }
+    catch (e) {
+        await supabaseAdmin
+            .from("floricode_sync_state")
+            .upsert({
+            resource: def.resource,
+            status: "error",
+            error_message: String(e?.message ?? e),
+            cursor_skip: skip,
+            updated_at: new Date().toISOString(),
+        }, { onConflict: "resource" });
+        throw e;
+    }
+    return {
+        ok: true,
+        resource: def.resource,
+        fetched,
+        upserted,
+        batches,
+        finished,
+        nextWatermark: latestChange,
+        durationMs: Date.now() - startedAt,
+    };
+}
+export async function syncAllFloricodeResources(opts) {
+    const results = [];
+    for (const def of FLORICODE_RESOURCES) {
+        try {
+            const r = await syncFloricodeResource(def.resource, {
+                batchSize: opts.batchSize,
+                maxBatches: opts.maxBatchesPerResource ?? 2,
+            });
+            results.push(r);
+        }
+        catch (e) {
+            results.push({ resource: def.resource, ok: false, error: String(e?.message ?? e) });
+        }
+    }
+    return { ok: true, results };
+}