@gscdump/engine-sqlite 0.6.3 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.mts CHANGED
@@ -1,10 +1,12 @@
1
1
  import { ResolverAdapter, SqlQuerySource, compileSqlite } from "@gscdump/engine/resolver";
2
2
  import { SQL, SQL as SQL$1, and, eq, gte, lte, sql } from "drizzle-orm";
3
3
  import * as _$drizzle_orm_sqlite_core0 from "drizzle-orm/sqlite-core";
4
+ import { ManifestStore } from "@gscdump/engine";
4
5
  import * as _$_gscdump_engine_scope0 from "@gscdump/engine/scope";
5
6
  import { ScopedRunnerOptions, TableScope } from "@gscdump/engine/scope";
6
7
  import { SqliteRemoteDatabase } from "drizzle-orm/sqlite-proxy";
7
8
  import { ComparisonMode, ResolveWindowOptions, ResolvedWindow, WindowPreset, resolveWindow } from "@gscdump/analysis/period";
9
+ import { DrizzleD1Database } from "drizzle-orm/d1";
8
10
  /**
9
11
  * Drizzle sqlite-core table definitions for the GSC analytics schema on D1.
10
12
  *
@@ -2277,6 +2279,888 @@ declare function aggClicks(t: MetricTable): SQL$1;
2277
2279
  declare function aggImpressions(t: MetricTable): SQL$1;
2278
2280
  declare function aggCtr(t: MetricTable): SQL$1;
2279
2281
  declare function aggPosition(t: MetricTable): SQL$1;
2282
+ declare const r2Manifest: _$drizzle_orm_sqlite_core0.SQLiteTableWithColumns<{
2283
+ name: "r2_manifest";
2284
+ schema: undefined;
2285
+ columns: {
2286
+ id: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2287
+ name: "id";
2288
+ tableName: "r2_manifest";
2289
+ dataType: "string";
2290
+ columnType: "SQLiteText";
2291
+ data: string;
2292
+ driverParam: string;
2293
+ notNull: true;
2294
+ hasDefault: false;
2295
+ isPrimaryKey: true;
2296
+ isAutoincrement: false;
2297
+ hasRuntimeDefault: false;
2298
+ enumValues: [string, ...string[]];
2299
+ baseColumn: never;
2300
+ identity: undefined;
2301
+ generated: undefined;
2302
+ }, {}, {
2303
+ length: number | undefined;
2304
+ }>;
2305
+ userId: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2306
+ name: "user_id";
2307
+ tableName: "r2_manifest";
2308
+ dataType: "number";
2309
+ columnType: "SQLiteInteger";
2310
+ data: number;
2311
+ driverParam: number;
2312
+ notNull: true;
2313
+ hasDefault: false;
2314
+ isPrimaryKey: false;
2315
+ isAutoincrement: false;
2316
+ hasRuntimeDefault: false;
2317
+ enumValues: undefined;
2318
+ baseColumn: never;
2319
+ identity: undefined;
2320
+ generated: undefined;
2321
+ }, {}, {}>;
2322
+ siteId: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2323
+ name: "site_id";
2324
+ tableName: "r2_manifest";
2325
+ dataType: "string";
2326
+ columnType: "SQLiteText";
2327
+ data: string;
2328
+ driverParam: string;
2329
+ notNull: false;
2330
+ hasDefault: false;
2331
+ isPrimaryKey: false;
2332
+ isAutoincrement: false;
2333
+ hasRuntimeDefault: false;
2334
+ enumValues: [string, ...string[]];
2335
+ baseColumn: never;
2336
+ identity: undefined;
2337
+ generated: undefined;
2338
+ }, {}, {
2339
+ length: number | undefined;
2340
+ }>;
2341
+ table: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2342
+ name: "table";
2343
+ tableName: "r2_manifest";
2344
+ dataType: "string";
2345
+ columnType: "SQLiteText";
2346
+ data: string;
2347
+ driverParam: string;
2348
+ notNull: true;
2349
+ hasDefault: false;
2350
+ isPrimaryKey: false;
2351
+ isAutoincrement: false;
2352
+ hasRuntimeDefault: false;
2353
+ enumValues: [string, ...string[]];
2354
+ baseColumn: never;
2355
+ identity: undefined;
2356
+ generated: undefined;
2357
+ }, {}, {
2358
+ length: number | undefined;
2359
+ }>;
2360
+ partition: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2361
+ name: "partition";
2362
+ tableName: "r2_manifest";
2363
+ dataType: "string";
2364
+ columnType: "SQLiteText";
2365
+ data: string;
2366
+ driverParam: string;
2367
+ notNull: true;
2368
+ hasDefault: false;
2369
+ isPrimaryKey: false;
2370
+ isAutoincrement: false;
2371
+ hasRuntimeDefault: false;
2372
+ enumValues: [string, ...string[]];
2373
+ baseColumn: never;
2374
+ identity: undefined;
2375
+ generated: undefined;
2376
+ }, {}, {
2377
+ length: number | undefined;
2378
+ }>;
2379
+ objectKey: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2380
+ name: "object_key";
2381
+ tableName: "r2_manifest";
2382
+ dataType: "string";
2383
+ columnType: "SQLiteText";
2384
+ data: string;
2385
+ driverParam: string;
2386
+ notNull: true;
2387
+ hasDefault: false;
2388
+ isPrimaryKey: false;
2389
+ isAutoincrement: false;
2390
+ hasRuntimeDefault: false;
2391
+ enumValues: [string, ...string[]];
2392
+ baseColumn: never;
2393
+ identity: undefined;
2394
+ generated: undefined;
2395
+ }, {}, {
2396
+ length: number | undefined;
2397
+ }>;
2398
+ rowCount: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2399
+ name: "row_count";
2400
+ tableName: "r2_manifest";
2401
+ dataType: "number";
2402
+ columnType: "SQLiteInteger";
2403
+ data: number;
2404
+ driverParam: number;
2405
+ notNull: true;
2406
+ hasDefault: true;
2407
+ isPrimaryKey: false;
2408
+ isAutoincrement: false;
2409
+ hasRuntimeDefault: false;
2410
+ enumValues: undefined;
2411
+ baseColumn: never;
2412
+ identity: undefined;
2413
+ generated: undefined;
2414
+ }, {}, {}>;
2415
+ bytes: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2416
+ name: "bytes";
2417
+ tableName: "r2_manifest";
2418
+ dataType: "number";
2419
+ columnType: "SQLiteInteger";
2420
+ data: number;
2421
+ driverParam: number;
2422
+ notNull: true;
2423
+ hasDefault: true;
2424
+ isPrimaryKey: false;
2425
+ isAutoincrement: false;
2426
+ hasRuntimeDefault: false;
2427
+ enumValues: undefined;
2428
+ baseColumn: never;
2429
+ identity: undefined;
2430
+ generated: undefined;
2431
+ }, {}, {}>;
2432
+ createdAt: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2433
+ name: "created_at";
2434
+ tableName: "r2_manifest";
2435
+ dataType: "number";
2436
+ columnType: "SQLiteInteger";
2437
+ data: number;
2438
+ driverParam: number;
2439
+ notNull: true;
2440
+ hasDefault: false;
2441
+ isPrimaryKey: false;
2442
+ isAutoincrement: false;
2443
+ hasRuntimeDefault: false;
2444
+ enumValues: undefined;
2445
+ baseColumn: never;
2446
+ identity: undefined;
2447
+ generated: undefined;
2448
+ }, {}, {}>;
2449
+ retiredAt: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2450
+ name: "retired_at";
2451
+ tableName: "r2_manifest";
2452
+ dataType: "number";
2453
+ columnType: "SQLiteInteger";
2454
+ data: number;
2455
+ driverParam: number;
2456
+ notNull: false;
2457
+ hasDefault: false;
2458
+ isPrimaryKey: false;
2459
+ isAutoincrement: false;
2460
+ hasRuntimeDefault: false;
2461
+ enumValues: undefined;
2462
+ baseColumn: never;
2463
+ identity: undefined;
2464
+ generated: undefined;
2465
+ }, {}, {}>;
2466
+ tier: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2467
+ name: "tier";
2468
+ tableName: "r2_manifest";
2469
+ dataType: "string";
2470
+ columnType: "SQLiteText";
2471
+ data: string;
2472
+ driverParam: string;
2473
+ notNull: false;
2474
+ hasDefault: false;
2475
+ isPrimaryKey: false;
2476
+ isAutoincrement: false;
2477
+ hasRuntimeDefault: false;
2478
+ enumValues: [string, ...string[]];
2479
+ baseColumn: never;
2480
+ identity: undefined;
2481
+ generated: undefined;
2482
+ }, {}, {
2483
+ length: number | undefined;
2484
+ }>;
2485
+ searchType: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2486
+ name: "search_type";
2487
+ tableName: "r2_manifest";
2488
+ dataType: "string";
2489
+ columnType: "SQLiteText";
2490
+ data: string;
2491
+ driverParam: string;
2492
+ notNull: false;
2493
+ hasDefault: false;
2494
+ isPrimaryKey: false;
2495
+ isAutoincrement: false;
2496
+ hasRuntimeDefault: false;
2497
+ enumValues: [string, ...string[]];
2498
+ baseColumn: never;
2499
+ identity: undefined;
2500
+ generated: undefined;
2501
+ }, {}, {
2502
+ length: number | undefined;
2503
+ }>;
2504
+ schemaVersion: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2505
+ name: "schema_version";
2506
+ tableName: "r2_manifest";
2507
+ dataType: "number";
2508
+ columnType: "SQLiteInteger";
2509
+ data: number;
2510
+ driverParam: number;
2511
+ notNull: false;
2512
+ hasDefault: false;
2513
+ isPrimaryKey: false;
2514
+ isAutoincrement: false;
2515
+ hasRuntimeDefault: false;
2516
+ enumValues: undefined;
2517
+ baseColumn: never;
2518
+ identity: undefined;
2519
+ generated: undefined;
2520
+ }, {}, {}>;
2521
+ };
2522
+ dialect: "sqlite";
2523
+ }>;
2524
+ type R2ManifestInsert = typeof r2Manifest.$inferInsert;
2525
+ type R2ManifestSelect = typeof r2Manifest.$inferSelect;
2526
+ declare const r2WriteErrors: _$drizzle_orm_sqlite_core0.SQLiteTableWithColumns<{
2527
+ name: "r2_write_errors";
2528
+ schema: undefined;
2529
+ columns: {
2530
+ id: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2531
+ name: "id";
2532
+ tableName: "r2_write_errors";
2533
+ dataType: "string";
2534
+ columnType: "SQLiteText";
2535
+ data: string;
2536
+ driverParam: string;
2537
+ notNull: true;
2538
+ hasDefault: false;
2539
+ isPrimaryKey: true;
2540
+ isAutoincrement: false;
2541
+ hasRuntimeDefault: false;
2542
+ enumValues: [string, ...string[]];
2543
+ baseColumn: never;
2544
+ identity: undefined;
2545
+ generated: undefined;
2546
+ }, {}, {
2547
+ length: number | undefined;
2548
+ }>;
2549
+ userId: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2550
+ name: "user_id";
2551
+ tableName: "r2_write_errors";
2552
+ dataType: "number";
2553
+ columnType: "SQLiteInteger";
2554
+ data: number;
2555
+ driverParam: number;
2556
+ notNull: true;
2557
+ hasDefault: false;
2558
+ isPrimaryKey: false;
2559
+ isAutoincrement: false;
2560
+ hasRuntimeDefault: false;
2561
+ enumValues: undefined;
2562
+ baseColumn: never;
2563
+ identity: undefined;
2564
+ generated: undefined;
2565
+ }, {}, {}>;
2566
+ siteId: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2567
+ name: "site_id";
2568
+ tableName: "r2_write_errors";
2569
+ dataType: "string";
2570
+ columnType: "SQLiteText";
2571
+ data: string;
2572
+ driverParam: string;
2573
+ notNull: false;
2574
+ hasDefault: false;
2575
+ isPrimaryKey: false;
2576
+ isAutoincrement: false;
2577
+ hasRuntimeDefault: false;
2578
+ enumValues: [string, ...string[]];
2579
+ baseColumn: never;
2580
+ identity: undefined;
2581
+ generated: undefined;
2582
+ }, {}, {
2583
+ length: number | undefined;
2584
+ }>;
2585
+ table: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2586
+ name: "table";
2587
+ tableName: "r2_write_errors";
2588
+ dataType: "string";
2589
+ columnType: "SQLiteText";
2590
+ data: string;
2591
+ driverParam: string;
2592
+ notNull: false;
2593
+ hasDefault: false;
2594
+ isPrimaryKey: false;
2595
+ isAutoincrement: false;
2596
+ hasRuntimeDefault: false;
2597
+ enumValues: [string, ...string[]];
2598
+ baseColumn: never;
2599
+ identity: undefined;
2600
+ generated: undefined;
2601
+ }, {}, {
2602
+ length: number | undefined;
2603
+ }>;
2604
+ date: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2605
+ name: "date";
2606
+ tableName: "r2_write_errors";
2607
+ dataType: "string";
2608
+ columnType: "SQLiteText";
2609
+ data: string;
2610
+ driverParam: string;
2611
+ notNull: false;
2612
+ hasDefault: false;
2613
+ isPrimaryKey: false;
2614
+ isAutoincrement: false;
2615
+ hasRuntimeDefault: false;
2616
+ enumValues: [string, ...string[]];
2617
+ baseColumn: never;
2618
+ identity: undefined;
2619
+ generated: undefined;
2620
+ }, {}, {
2621
+ length: number | undefined;
2622
+ }>;
2623
+ error: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2624
+ name: "error";
2625
+ tableName: "r2_write_errors";
2626
+ dataType: "string";
2627
+ columnType: "SQLiteText";
2628
+ data: string;
2629
+ driverParam: string;
2630
+ notNull: true;
2631
+ hasDefault: false;
2632
+ isPrimaryKey: false;
2633
+ isAutoincrement: false;
2634
+ hasRuntimeDefault: false;
2635
+ enumValues: [string, ...string[]];
2636
+ baseColumn: never;
2637
+ identity: undefined;
2638
+ generated: undefined;
2639
+ }, {}, {
2640
+ length: number | undefined;
2641
+ }>;
2642
+ createdAt: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2643
+ name: "created_at";
2644
+ tableName: "r2_write_errors";
2645
+ dataType: "number";
2646
+ columnType: "SQLiteInteger";
2647
+ data: number;
2648
+ driverParam: number;
2649
+ notNull: true;
2650
+ hasDefault: true;
2651
+ isPrimaryKey: false;
2652
+ isAutoincrement: false;
2653
+ hasRuntimeDefault: false;
2654
+ enumValues: undefined;
2655
+ baseColumn: never;
2656
+ identity: undefined;
2657
+ generated: undefined;
2658
+ }, {}, {}>;
2659
+ };
2660
+ dialect: "sqlite";
2661
+ }>;
2662
+ type R2WriteErrorInsert = typeof r2WriteErrors.$inferInsert;
2663
+ type R2WriteErrorSelect = typeof r2WriteErrors.$inferSelect;
2664
+ declare const r2ShadowDiffs: _$drizzle_orm_sqlite_core0.SQLiteTableWithColumns<{
2665
+ name: "r2_shadow_diffs";
2666
+ schema: undefined;
2667
+ columns: {
2668
+ id: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2669
+ name: "id";
2670
+ tableName: "r2_shadow_diffs";
2671
+ dataType: "string";
2672
+ columnType: "SQLiteText";
2673
+ data: string;
2674
+ driverParam: string;
2675
+ notNull: true;
2676
+ hasDefault: false;
2677
+ isPrimaryKey: true;
2678
+ isAutoincrement: false;
2679
+ hasRuntimeDefault: false;
2680
+ enumValues: [string, ...string[]];
2681
+ baseColumn: never;
2682
+ identity: undefined;
2683
+ generated: undefined;
2684
+ }, {}, {
2685
+ length: number | undefined;
2686
+ }>;
2687
+ userId: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2688
+ name: "user_id";
2689
+ tableName: "r2_shadow_diffs";
2690
+ dataType: "number";
2691
+ columnType: "SQLiteInteger";
2692
+ data: number;
2693
+ driverParam: number;
2694
+ notNull: true;
2695
+ hasDefault: false;
2696
+ isPrimaryKey: false;
2697
+ isAutoincrement: false;
2698
+ hasRuntimeDefault: false;
2699
+ enumValues: undefined;
2700
+ baseColumn: never;
2701
+ identity: undefined;
2702
+ generated: undefined;
2703
+ }, {}, {}>;
2704
+ siteId: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2705
+ name: "site_id";
2706
+ tableName: "r2_shadow_diffs";
2707
+ dataType: "string";
2708
+ columnType: "SQLiteText";
2709
+ data: string;
2710
+ driverParam: string;
2711
+ notNull: false;
2712
+ hasDefault: false;
2713
+ isPrimaryKey: false;
2714
+ isAutoincrement: false;
2715
+ hasRuntimeDefault: false;
2716
+ enumValues: [string, ...string[]];
2717
+ baseColumn: never;
2718
+ identity: undefined;
2719
+ generated: undefined;
2720
+ }, {}, {
2721
+ length: number | undefined;
2722
+ }>;
2723
+ endpoint: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2724
+ name: "endpoint";
2725
+ tableName: "r2_shadow_diffs";
2726
+ dataType: "string";
2727
+ columnType: "SQLiteText";
2728
+ data: string;
2729
+ driverParam: string;
2730
+ notNull: true;
2731
+ hasDefault: false;
2732
+ isPrimaryKey: false;
2733
+ isAutoincrement: false;
2734
+ hasRuntimeDefault: false;
2735
+ enumValues: [string, ...string[]];
2736
+ baseColumn: never;
2737
+ identity: undefined;
2738
+ generated: undefined;
2739
+ }, {}, {
2740
+ length: number | undefined;
2741
+ }>;
2742
+ diff: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2743
+ name: "diff";
2744
+ tableName: "r2_shadow_diffs";
2745
+ dataType: "string";
2746
+ columnType: "SQLiteText";
2747
+ data: string;
2748
+ driverParam: string;
2749
+ notNull: true;
2750
+ hasDefault: false;
2751
+ isPrimaryKey: false;
2752
+ isAutoincrement: false;
2753
+ hasRuntimeDefault: false;
2754
+ enumValues: [string, ...string[]];
2755
+ baseColumn: never;
2756
+ identity: undefined;
2757
+ generated: undefined;
2758
+ }, {}, {
2759
+ length: number | undefined;
2760
+ }>;
2761
+ createdAt: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2762
+ name: "created_at";
2763
+ tableName: "r2_shadow_diffs";
2764
+ dataType: "number";
2765
+ columnType: "SQLiteInteger";
2766
+ data: number;
2767
+ driverParam: number;
2768
+ notNull: true;
2769
+ hasDefault: true;
2770
+ isPrimaryKey: false;
2771
+ isAutoincrement: false;
2772
+ hasRuntimeDefault: false;
2773
+ enumValues: undefined;
2774
+ baseColumn: never;
2775
+ identity: undefined;
2776
+ generated: undefined;
2777
+ }, {}, {}>;
2778
+ };
2779
+ dialect: "sqlite";
2780
+ }>;
2781
+ type R2ShadowDiffInsert = typeof r2ShadowDiffs.$inferInsert;
2782
+ type R2ShadowDiffSelect = typeof r2ShadowDiffs.$inferSelect;
2783
+ declare const r2Locks: _$drizzle_orm_sqlite_core0.SQLiteTableWithColumns<{
2784
+ name: "r2_locks";
2785
+ schema: undefined;
2786
+ columns: {
2787
+ scope: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2788
+ name: "scope";
2789
+ tableName: "r2_locks";
2790
+ dataType: "string";
2791
+ columnType: "SQLiteText";
2792
+ data: string;
2793
+ driverParam: string;
2794
+ notNull: true;
2795
+ hasDefault: false;
2796
+ isPrimaryKey: true;
2797
+ isAutoincrement: false;
2798
+ hasRuntimeDefault: false;
2799
+ enumValues: [string, ...string[]];
2800
+ baseColumn: never;
2801
+ identity: undefined;
2802
+ generated: undefined;
2803
+ }, {}, {
2804
+ length: number | undefined;
2805
+ }>;
2806
+ holderId: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2807
+ name: "holder_id";
2808
+ tableName: "r2_locks";
2809
+ dataType: "string";
2810
+ columnType: "SQLiteText";
2811
+ data: string;
2812
+ driverParam: string;
2813
+ notNull: true;
2814
+ hasDefault: false;
2815
+ isPrimaryKey: false;
2816
+ isAutoincrement: false;
2817
+ hasRuntimeDefault: false;
2818
+ enumValues: [string, ...string[]];
2819
+ baseColumn: never;
2820
+ identity: undefined;
2821
+ generated: undefined;
2822
+ }, {}, {
2823
+ length: number | undefined;
2824
+ }>;
2825
+ acquiredAt: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2826
+ name: "acquired_at";
2827
+ tableName: "r2_locks";
2828
+ dataType: "number";
2829
+ columnType: "SQLiteInteger";
2830
+ data: number;
2831
+ driverParam: number;
2832
+ notNull: true;
2833
+ hasDefault: false;
2834
+ isPrimaryKey: false;
2835
+ isAutoincrement: false;
2836
+ hasRuntimeDefault: false;
2837
+ enumValues: undefined;
2838
+ baseColumn: never;
2839
+ identity: undefined;
2840
+ generated: undefined;
2841
+ }, {}, {}>;
2842
+ expiresAt: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2843
+ name: "expires_at";
2844
+ tableName: "r2_locks";
2845
+ dataType: "number";
2846
+ columnType: "SQLiteInteger";
2847
+ data: number;
2848
+ driverParam: number;
2849
+ notNull: true;
2850
+ hasDefault: false;
2851
+ isPrimaryKey: false;
2852
+ isAutoincrement: false;
2853
+ hasRuntimeDefault: false;
2854
+ enumValues: undefined;
2855
+ baseColumn: never;
2856
+ identity: undefined;
2857
+ generated: undefined;
2858
+ }, {}, {}>;
2859
+ };
2860
+ dialect: "sqlite";
2861
+ }>;
2862
+ type R2LockInsert = typeof r2Locks.$inferInsert;
2863
+ type R2LockSelect = typeof r2Locks.$inferSelect;
2864
+ declare const r2Watermarks: _$drizzle_orm_sqlite_core0.SQLiteTableWithColumns<{
2865
+ name: "r2_watermarks";
2866
+ schema: undefined;
2867
+ columns: {
2868
+ userId: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2869
+ name: "user_id";
2870
+ tableName: "r2_watermarks";
2871
+ dataType: "number";
2872
+ columnType: "SQLiteInteger";
2873
+ data: number;
2874
+ driverParam: number;
2875
+ notNull: true;
2876
+ hasDefault: false;
2877
+ isPrimaryKey: false;
2878
+ isAutoincrement: false;
2879
+ hasRuntimeDefault: false;
2880
+ enumValues: undefined;
2881
+ baseColumn: never;
2882
+ identity: undefined;
2883
+ generated: undefined;
2884
+ }, {}, {}>;
2885
+ siteId: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2886
+ name: "site_id";
2887
+ tableName: "r2_watermarks";
2888
+ dataType: "string";
2889
+ columnType: "SQLiteText";
2890
+ data: string;
2891
+ driverParam: string;
2892
+ notNull: true;
2893
+ hasDefault: true;
2894
+ isPrimaryKey: false;
2895
+ isAutoincrement: false;
2896
+ hasRuntimeDefault: false;
2897
+ enumValues: [string, ...string[]];
2898
+ baseColumn: never;
2899
+ identity: undefined;
2900
+ generated: undefined;
2901
+ }, {}, {
2902
+ length: number | undefined;
2903
+ }>;
2904
+ table: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2905
+ name: "table";
2906
+ tableName: "r2_watermarks";
2907
+ dataType: "string";
2908
+ columnType: "SQLiteText";
2909
+ data: string;
2910
+ driverParam: string;
2911
+ notNull: true;
2912
+ hasDefault: false;
2913
+ isPrimaryKey: false;
2914
+ isAutoincrement: false;
2915
+ hasRuntimeDefault: false;
2916
+ enumValues: [string, ...string[]];
2917
+ baseColumn: never;
2918
+ identity: undefined;
2919
+ generated: undefined;
2920
+ }, {}, {
2921
+ length: number | undefined;
2922
+ }>;
2923
+ newestDateSynced: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2924
+ name: "newest_date_synced";
2925
+ tableName: "r2_watermarks";
2926
+ dataType: "string";
2927
+ columnType: "SQLiteText";
2928
+ data: string;
2929
+ driverParam: string;
2930
+ notNull: true;
2931
+ hasDefault: false;
2932
+ isPrimaryKey: false;
2933
+ isAutoincrement: false;
2934
+ hasRuntimeDefault: false;
2935
+ enumValues: [string, ...string[]];
2936
+ baseColumn: never;
2937
+ identity: undefined;
2938
+ generated: undefined;
2939
+ }, {}, {
2940
+ length: number | undefined;
2941
+ }>;
2942
+ oldestDateSynced: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2943
+ name: "oldest_date_synced";
2944
+ tableName: "r2_watermarks";
2945
+ dataType: "string";
2946
+ columnType: "SQLiteText";
2947
+ data: string;
2948
+ driverParam: string;
2949
+ notNull: true;
2950
+ hasDefault: false;
2951
+ isPrimaryKey: false;
2952
+ isAutoincrement: false;
2953
+ hasRuntimeDefault: false;
2954
+ enumValues: [string, ...string[]];
2955
+ baseColumn: never;
2956
+ identity: undefined;
2957
+ generated: undefined;
2958
+ }, {}, {
2959
+ length: number | undefined;
2960
+ }>;
2961
+ lastSyncAt: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2962
+ name: "last_sync_at";
2963
+ tableName: "r2_watermarks";
2964
+ dataType: "number";
2965
+ columnType: "SQLiteInteger";
2966
+ data: number;
2967
+ driverParam: number;
2968
+ notNull: true;
2969
+ hasDefault: false;
2970
+ isPrimaryKey: false;
2971
+ isAutoincrement: false;
2972
+ hasRuntimeDefault: false;
2973
+ enumValues: undefined;
2974
+ baseColumn: never;
2975
+ identity: undefined;
2976
+ generated: undefined;
2977
+ }, {}, {}>;
2978
+ };
2979
+ dialect: "sqlite";
2980
+ }>;
2981
+ type R2WatermarkInsert = typeof r2Watermarks.$inferInsert;
2982
+ type R2WatermarkSelect = typeof r2Watermarks.$inferSelect;
2983
+ declare const r2SyncStates: _$drizzle_orm_sqlite_core0.SQLiteTableWithColumns<{
2984
+ name: "r2_sync_states";
2985
+ schema: undefined;
2986
+ columns: {
2987
+ userId: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
2988
+ name: "user_id";
2989
+ tableName: "r2_sync_states";
2990
+ dataType: "number";
2991
+ columnType: "SQLiteInteger";
2992
+ data: number;
2993
+ driverParam: number;
2994
+ notNull: true;
2995
+ hasDefault: false;
2996
+ isPrimaryKey: false;
2997
+ isAutoincrement: false;
2998
+ hasRuntimeDefault: false;
2999
+ enumValues: undefined;
3000
+ baseColumn: never;
3001
+ identity: undefined;
3002
+ generated: undefined;
3003
+ }, {}, {}>;
3004
+ siteId: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
3005
+ name: "site_id";
3006
+ tableName: "r2_sync_states";
3007
+ dataType: "string";
3008
+ columnType: "SQLiteText";
3009
+ data: string;
3010
+ driverParam: string;
3011
+ notNull: true;
3012
+ hasDefault: true;
3013
+ isPrimaryKey: false;
3014
+ isAutoincrement: false;
3015
+ hasRuntimeDefault: false;
3016
+ enumValues: [string, ...string[]];
3017
+ baseColumn: never;
3018
+ identity: undefined;
3019
+ generated: undefined;
3020
+ }, {}, {
3021
+ length: number | undefined;
3022
+ }>;
3023
+ table: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
3024
+ name: "table";
3025
+ tableName: "r2_sync_states";
3026
+ dataType: "string";
3027
+ columnType: "SQLiteText";
3028
+ data: string;
3029
+ driverParam: string;
3030
+ notNull: true;
3031
+ hasDefault: false;
3032
+ isPrimaryKey: false;
3033
+ isAutoincrement: false;
3034
+ hasRuntimeDefault: false;
3035
+ enumValues: [string, ...string[]];
3036
+ baseColumn: never;
3037
+ identity: undefined;
3038
+ generated: undefined;
3039
+ }, {}, {
3040
+ length: number | undefined;
3041
+ }>;
3042
+ date: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
3043
+ name: "date";
3044
+ tableName: "r2_sync_states";
3045
+ dataType: "string";
3046
+ columnType: "SQLiteText";
3047
+ data: string;
3048
+ driverParam: string;
3049
+ notNull: true;
3050
+ hasDefault: false;
3051
+ isPrimaryKey: false;
3052
+ isAutoincrement: false;
3053
+ hasRuntimeDefault: false;
3054
+ enumValues: [string, ...string[]];
3055
+ baseColumn: never;
3056
+ identity: undefined;
3057
+ generated: undefined;
3058
+ }, {}, {
3059
+ length: number | undefined;
3060
+ }>;
3061
+ searchType: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
3062
+ name: "search_type";
3063
+ tableName: "r2_sync_states";
3064
+ dataType: "string";
3065
+ columnType: "SQLiteText";
3066
+ data: string;
3067
+ driverParam: string;
3068
+ notNull: true;
3069
+ hasDefault: true;
3070
+ isPrimaryKey: false;
3071
+ isAutoincrement: false;
3072
+ hasRuntimeDefault: false;
3073
+ enumValues: [string, ...string[]];
3074
+ baseColumn: never;
3075
+ identity: undefined;
3076
+ generated: undefined;
3077
+ }, {}, {
3078
+ length: number | undefined;
3079
+ }>;
3080
+ state: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
3081
+ name: "state";
3082
+ tableName: "r2_sync_states";
3083
+ dataType: "string";
3084
+ columnType: "SQLiteText";
3085
+ data: "pending" | "inflight" | "done" | "failed";
3086
+ driverParam: string;
3087
+ notNull: true;
3088
+ hasDefault: false;
3089
+ isPrimaryKey: false;
3090
+ isAutoincrement: false;
3091
+ hasRuntimeDefault: false;
3092
+ enumValues: ["pending", "inflight", "done", "failed"];
3093
+ baseColumn: never;
3094
+ identity: undefined;
3095
+ generated: undefined;
3096
+ }, {}, {
3097
+ length: number | undefined;
3098
+ }>;
3099
+ updatedAt: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
3100
+ name: "updated_at";
3101
+ tableName: "r2_sync_states";
3102
+ dataType: "number";
3103
+ columnType: "SQLiteInteger";
3104
+ data: number;
3105
+ driverParam: number;
3106
+ notNull: true;
3107
+ hasDefault: false;
3108
+ isPrimaryKey: false;
3109
+ isAutoincrement: false;
3110
+ hasRuntimeDefault: false;
3111
+ enumValues: undefined;
3112
+ baseColumn: never;
3113
+ identity: undefined;
3114
+ generated: undefined;
3115
+ }, {}, {}>;
3116
+ attempts: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
3117
+ name: "attempts";
3118
+ tableName: "r2_sync_states";
3119
+ dataType: "number";
3120
+ columnType: "SQLiteInteger";
3121
+ data: number;
3122
+ driverParam: number;
3123
+ notNull: true;
3124
+ hasDefault: true;
3125
+ isPrimaryKey: false;
3126
+ isAutoincrement: false;
3127
+ hasRuntimeDefault: false;
3128
+ enumValues: undefined;
3129
+ baseColumn: never;
3130
+ identity: undefined;
3131
+ generated: undefined;
3132
+ }, {}, {}>;
3133
+ error: _$drizzle_orm_sqlite_core0.SQLiteColumn<{
3134
+ name: "error";
3135
+ tableName: "r2_sync_states";
3136
+ dataType: "string";
3137
+ columnType: "SQLiteText";
3138
+ data: string;
3139
+ driverParam: string;
3140
+ notNull: false;
3141
+ hasDefault: false;
3142
+ isPrimaryKey: false;
3143
+ isAutoincrement: false;
3144
+ hasRuntimeDefault: false;
3145
+ enumValues: [string, ...string[]];
3146
+ baseColumn: never;
3147
+ identity: undefined;
3148
+ generated: undefined;
3149
+ }, {}, {
3150
+ length: number | undefined;
3151
+ }>;
3152
+ };
3153
+ dialect: "sqlite";
3154
+ }>;
3155
+ type R2SyncStateInsert = typeof r2SyncStates.$inferInsert;
3156
+ type R2SyncStateSelect = typeof r2SyncStates.$inferSelect;
3157
+ type AnalyticsManifestDb = DrizzleD1Database<{
3158
+ r2Manifest: typeof r2Manifest;
3159
+ r2Locks: typeof r2Locks;
3160
+ r2SyncStates: typeof r2SyncStates;
3161
+ r2Watermarks: typeof r2Watermarks;
3162
+ }>;
3163
+ declare function createD1ManifestStore(db: AnalyticsManifestDb): ManifestStore;
2280
3164
  type TableKey = keyof typeof schema;
2281
3165
  interface CreateSqliteResolverAdapterOptions {
2282
3166
  regex?: boolean;
@@ -2300,4 +3184,4 @@ interface CreateSqliteResolverAdapterFromExecutorOptions {
2300
3184
  * an executor handy.
2301
3185
  */
2302
3186
  declare function createSqliteResolverAdapterFromExecutor(options: CreateSqliteResolverAdapterFromExecutorOptions): Promise<ResolverAdapter<TableKey>>;
2303
- export { type ComparisonMode, type EngineConfig, type ResolveWindowOptions, type ResolvedWindow, type SQL, type Schema, type ScopedRunnerOptions, type SqliteInsightRunner, type SqliteInsightRunnerOptions, type SqliteQueryExecutor, type SqliteRowExecutor, type TableKey, type TableScope, type WindowPreset, aggClicks, aggCtr, aggImpressions, aggPosition, and, compileSqlite, createEngine, createSqliteInsightRunner, createSqliteResolverAdapter, createSqliteResolverAdapterFromExecutor, eq, gsc_countries, gsc_devices, gsc_keywords, gsc_page_keywords, gsc_pages, gte, lte, mergeScope, probeSqliteRegex, resolveWindow, schema, scopeFor, sql, sqliteResolverAdapter };
3187
+ export { type AnalyticsManifestDb, type ComparisonMode, type EngineConfig, type R2LockInsert, type R2LockSelect, type R2ManifestInsert, type R2ManifestSelect, type R2ShadowDiffInsert, type R2ShadowDiffSelect, type R2SyncStateInsert, type R2SyncStateSelect, type R2WatermarkInsert, type R2WatermarkSelect, type R2WriteErrorInsert, type R2WriteErrorSelect, type ResolveWindowOptions, type ResolvedWindow, type SQL, type Schema, type ScopedRunnerOptions, type SqliteInsightRunner, type SqliteInsightRunnerOptions, type SqliteQueryExecutor, type SqliteRowExecutor, type TableKey, type TableScope, type WindowPreset, aggClicks, aggCtr, aggImpressions, aggPosition, and, compileSqlite, createD1ManifestStore, createEngine, createSqliteInsightRunner, createSqliteResolverAdapter, createSqliteResolverAdapterFromExecutor, eq, gsc_countries, gsc_devices, gsc_keywords, gsc_page_keywords, gsc_pages, gte, lte, mergeScope, probeSqliteRegex, r2Locks, r2Manifest, r2ShadowDiffs, r2SyncStates, r2Watermarks, r2WriteErrors, resolveWindow, schema, scopeFor, sql, sqliteResolverAdapter };
package/dist/index.mjs CHANGED
@@ -1,6 +1,7 @@
1
1
  import { assertSchemaInSync, compileSqlite, compileSqlite as compileSqlite$1, createResolverAdapter, createSqlQuerySource } from "@gscdump/engine/resolver";
2
- import { and, eq, gte, lte, sql, sql as sql$1 } from "drizzle-orm";
3
- import { integer, real, sqliteTable, text } from "drizzle-orm/sqlite-core";
2
+ import { and, and as and$1, eq, eq as eq$1, gte, inArray, isNotNull, isNull, lt, lte, lte as lte$1, or, sql, sql as sql$1 } from "drizzle-orm";
3
+ import { index, integer, primaryKey, real, sqliteTable, text, unique } from "drizzle-orm/sqlite-core";
4
+ import { inferSearchType } from "@gscdump/engine";
4
5
  import { createScopedHelpers } from "@gscdump/engine/scope";
5
6
  import { drizzle } from "drizzle-orm/sqlite-proxy";
6
7
  import { resolveWindow } from "@gscdump/analysis/period";
@@ -121,6 +122,346 @@ function aggCtr(t) {
121
122
  function aggPosition(t) {
122
123
  return sql$1`SUM(${t.sum_position}) / NULLIF(SUM(${t.impressions}), 0) + 1`;
123
124
  }
125
+ const r2Manifest = sqliteTable("r2_manifest", {
126
+ id: text("id").primaryKey(),
127
+ userId: integer("user_id").notNull(),
128
+ siteId: text("site_id"),
129
+ table: text("table").notNull(),
130
+ partition: text("partition").notNull(),
131
+ objectKey: text("object_key").notNull(),
132
+ rowCount: integer("row_count").notNull().default(0),
133
+ bytes: integer("bytes").notNull().default(0),
134
+ createdAt: integer("created_at").notNull(),
135
+ retiredAt: integer("retired_at"),
136
+ tier: text("tier"),
137
+ searchType: text("search_type"),
138
+ schemaVersion: integer("schema_version")
139
+ }, (t) => [
140
+ index("idx_r2_manifest_live").on(t.userId, t.siteId, t.table, t.partition, t.retiredAt),
141
+ index("idx_r2_manifest_retired").on(t.retiredAt),
142
+ index("idx_r2_manifest_tier").on(t.userId, t.siteId, t.table, t.tier, t.retiredAt),
143
+ unique("r2_manifest_object_key_unique").on(t.objectKey)
144
+ ]);
145
+ const r2WriteErrors = sqliteTable("r2_write_errors", {
146
+ id: text("id").primaryKey(),
147
+ userId: integer("user_id").notNull(),
148
+ siteId: text("site_id"),
149
+ table: text("table"),
150
+ date: text("date"),
151
+ error: text("error").notNull(),
152
+ createdAt: integer("created_at").notNull().default(sql$1`(unixepoch())`)
153
+ }, (t) => [index("idx_r2_write_errors_user").on(t.userId, t.createdAt), index("idx_r2_write_errors_created").on(t.createdAt)]);
154
+ const r2ShadowDiffs = sqliteTable("r2_shadow_diffs", {
155
+ id: text("id").primaryKey(),
156
+ userId: integer("user_id").notNull(),
157
+ siteId: text("site_id"),
158
+ endpoint: text("endpoint").notNull(),
159
+ diff: text("diff").notNull(),
160
+ createdAt: integer("created_at").notNull().default(sql$1`(unixepoch())`)
161
+ }, (t) => [index("idx_r2_shadow_diffs_user").on(t.userId, t.createdAt), index("idx_r2_shadow_diffs_endpoint").on(t.endpoint, t.createdAt)]);
162
+ const r2Locks = sqliteTable("r2_locks", {
163
+ scope: text("scope").primaryKey(),
164
+ holderId: text("holder_id").notNull(),
165
+ acquiredAt: integer("acquired_at").notNull(),
166
+ expiresAt: integer("expires_at").notNull()
167
+ }, (t) => [index("idx_r2_locks_expires").on(t.expiresAt)]);
168
+ const r2Watermarks = sqliteTable("r2_watermarks", {
169
+ userId: integer("user_id").notNull(),
170
+ siteId: text("site_id").notNull().default(""),
171
+ table: text("table").notNull(),
172
+ newestDateSynced: text("newest_date_synced").notNull(),
173
+ oldestDateSynced: text("oldest_date_synced").notNull(),
174
+ lastSyncAt: integer("last_sync_at").notNull()
175
+ }, (t) => [primaryKey({ columns: [
176
+ t.userId,
177
+ t.siteId,
178
+ t.table
179
+ ] })]);
180
+ const r2SyncStates = sqliteTable("r2_sync_states", {
181
+ userId: integer("user_id").notNull(),
182
+ siteId: text("site_id").notNull().default(""),
183
+ table: text("table").notNull(),
184
+ date: text("date").notNull(),
185
+ searchType: text("search_type").notNull().default(""),
186
+ state: text("state", { enum: [
187
+ "pending",
188
+ "inflight",
189
+ "done",
190
+ "failed"
191
+ ] }).notNull(),
192
+ updatedAt: integer("updated_at").notNull(),
193
+ attempts: integer("attempts").notNull().default(0),
194
+ error: text("error")
195
+ }, (t) => [primaryKey({ columns: [
196
+ t.userId,
197
+ t.siteId,
198
+ t.table,
199
+ t.date,
200
+ t.searchType
201
+ ] }), index("idx_r2_sync_states_state").on(t.state)]);
202
+ function toRow(e) {
203
+ return {
204
+ id: e.objectKey,
205
+ userId: Number(e.userId),
206
+ siteId: e.siteId ?? null,
207
+ table: e.table,
208
+ partition: e.partition,
209
+ objectKey: e.objectKey,
210
+ rowCount: e.rowCount,
211
+ bytes: e.bytes,
212
+ createdAt: e.createdAt,
213
+ retiredAt: e.retiredAt ?? null,
214
+ tier: e.tier ?? null,
215
+ searchType: e.searchType ?? null,
216
+ schemaVersion: e.schemaVersion ?? null
217
+ };
218
+ }
219
+ function fromRow(r) {
220
+ return {
221
+ userId: String(r.userId),
222
+ siteId: r.siteId ?? void 0,
223
+ table: r.table,
224
+ partition: r.partition,
225
+ objectKey: r.objectKey,
226
+ rowCount: r.rowCount,
227
+ bytes: r.bytes,
228
+ createdAt: r.createdAt,
229
+ retiredAt: r.retiredAt ?? void 0,
230
+ ...r.tier !== null ? { tier: r.tier } : {},
231
+ ...r.searchType !== null ? { searchType: r.searchType } : {},
232
+ ...r.schemaVersion !== null ? { schemaVersion: r.schemaVersion } : {}
233
+ };
234
+ }
235
+ const siteIdOf = (s) => s ?? "";
236
+ const searchTypeOf = (s) => s === void 0 || s === "web" ? "" : s;
237
+ const LOCK_TTL_MS = 3e4;
238
+ const LOCK_ACQUIRE_TIMEOUT_MS = 5e3;
239
+ const LOCK_RETRY_MIN_MS = 25;
240
+ const LOCK_RETRY_MAX_MS = 150;
241
+ function lockScopeKey(scope) {
242
+ return `${scope.userId}|${siteIdOf(scope.siteId)}|${scope.table}|${scope.partition}`;
243
+ }
244
+ function jitterDelay() {
245
+ return LOCK_RETRY_MIN_MS + Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
246
+ }
247
+ function tierMatchCond(target) {
248
+ const explicit = eq$1(r2Manifest.tier, target);
249
+ if (target === "raw") return or(explicit, and$1(isNull(r2Manifest.tier), sql$1`${r2Manifest.partition} LIKE 'daily/%'`));
250
+ if (target === "d30") return or(explicit, and$1(isNull(r2Manifest.tier), sql$1`${r2Manifest.partition} LIKE 'monthly/%'`));
251
+ return explicit;
252
+ }
253
+ function createD1ManifestStore(db) {
254
+ async function listByFilter(filter, liveOnly) {
255
+ const conds = [eq$1(r2Manifest.userId, Number(filter.userId))];
256
+ if (liveOnly) conds.push(isNull(r2Manifest.retiredAt));
257
+ if (filter.siteId !== void 0) conds.push(eq$1(r2Manifest.siteId, filter.siteId));
258
+ if (filter.table !== void 0) conds.push(eq$1(r2Manifest.table, filter.table));
259
+ if (filter.partitions && filter.partitions.length > 0) conds.push(inArray(r2Manifest.partition, filter.partitions));
260
+ if (filter.tier !== void 0) {
261
+ const cond = tierMatchCond(filter.tier);
262
+ if (cond) conds.push(cond);
263
+ }
264
+ return (await db.select().from(r2Manifest).where(and$1(...conds))).map(fromRow);
265
+ }
266
+ const listLive = (filter) => listByFilter(filter, true);
267
+ const listAll = (filter) => listByFilter(filter, false);
268
+ async function registerVersions(newEntries, superseding) {
269
+ const supersededAt = newEntries[0]?.createdAt ?? Date.now();
270
+ const statements = [];
271
+ if (superseding && superseding.length > 0) {
272
+ const keys = superseding.map((s) => s.objectKey);
273
+ const CHUNK = 90;
274
+ for (let i = 0; i < keys.length; i += CHUNK) {
275
+ const slice = keys.slice(i, i + CHUNK);
276
+ statements.push(db.update(r2Manifest).set({ retiredAt: supersededAt }).where(and$1(inArray(r2Manifest.objectKey, slice), isNull(r2Manifest.retiredAt))));
277
+ }
278
+ }
279
+ for (const e of newEntries) statements.push(db.insert(r2Manifest).values(toRow(e)).onConflictDoUpdate({
280
+ target: r2Manifest.objectKey,
281
+ set: {
282
+ userId: sql$1`excluded.user_id`,
283
+ siteId: sql$1`excluded.site_id`,
284
+ table: sql$1`excluded."table"`,
285
+ partition: sql$1`excluded.partition`,
286
+ rowCount: sql$1`excluded.row_count`,
287
+ bytes: sql$1`excluded.bytes`,
288
+ createdAt: sql$1`excluded.created_at`,
289
+ retiredAt: sql$1`excluded.retired_at`,
290
+ tier: sql$1`excluded.tier`,
291
+ searchType: sql$1`excluded.search_type`,
292
+ schemaVersion: sql$1`excluded.schema_version`
293
+ }
294
+ }));
295
+ if (statements.length === 0) return;
296
+ const BATCH_LIMIT = 95;
297
+ for (let i = 0; i < statements.length; i += BATCH_LIMIT) {
298
+ const chunk = statements.slice(i, i + BATCH_LIMIT);
299
+ await db.batch(chunk);
300
+ }
301
+ }
302
+ async function listRetired(olderThan) {
303
+ return (await db.select().from(r2Manifest).where(and$1(isNotNull(r2Manifest.retiredAt), lte$1(r2Manifest.retiredAt, olderThan)))).map(fromRow);
304
+ }
305
+ async function deleteEntries(entries) {
306
+ if (entries.length === 0) return;
307
+ const keys = entries.map((e) => e.objectKey);
308
+ const CHUNK = 90;
309
+ for (let i = 0; i < keys.length; i += CHUNK) await db.delete(r2Manifest).where(inArray(r2Manifest.objectKey, keys.slice(i, i + CHUNK)));
310
+ }
311
+ async function getWatermarks(filter) {
312
+ const conds = [eq$1(r2Watermarks.userId, Number(filter.userId))];
313
+ if (filter.siteId !== void 0) conds.push(eq$1(r2Watermarks.siteId, filter.siteId));
314
+ if (filter.table !== void 0) conds.push(eq$1(r2Watermarks.table, filter.table));
315
+ return (await db.select().from(r2Watermarks).where(and$1(...conds))).map((r) => ({
316
+ userId: String(r.userId),
317
+ siteId: r.siteId === "" ? void 0 : r.siteId,
318
+ table: r.table,
319
+ newestDateSynced: r.newestDateSynced,
320
+ oldestDateSynced: r.oldestDateSynced,
321
+ lastSyncAt: r.lastSyncAt
322
+ }));
323
+ }
324
+ async function bumpWatermark(scope, date, at) {
325
+ const lastSyncAt = at ?? Date.now();
326
+ await db.insert(r2Watermarks).values({
327
+ userId: Number(scope.userId),
328
+ siteId: siteIdOf(scope.siteId),
329
+ table: scope.table,
330
+ newestDateSynced: date,
331
+ oldestDateSynced: date,
332
+ lastSyncAt
333
+ }).onConflictDoUpdate({
334
+ target: [
335
+ r2Watermarks.userId,
336
+ r2Watermarks.siteId,
337
+ r2Watermarks.table
338
+ ],
339
+ set: {
340
+ newestDateSynced: sql$1`CASE WHEN excluded.newest_date_synced > newest_date_synced THEN excluded.newest_date_synced ELSE newest_date_synced END`,
341
+ oldestDateSynced: sql$1`CASE WHEN excluded.oldest_date_synced < oldest_date_synced THEN excluded.oldest_date_synced ELSE oldest_date_synced END`,
342
+ lastSyncAt: sql$1`excluded.last_sync_at`
343
+ }
344
+ }).run();
345
+ }
346
+ async function getSyncStates(filter) {
347
+ const conds = [eq$1(r2SyncStates.userId, Number(filter.userId))];
348
+ if (filter.siteId !== void 0) conds.push(eq$1(r2SyncStates.siteId, filter.siteId));
349
+ if (filter.table !== void 0) conds.push(eq$1(r2SyncStates.table, filter.table));
350
+ if (filter.state !== void 0) conds.push(eq$1(r2SyncStates.state, filter.state));
351
+ if (filter.searchType !== void 0) conds.push(eq$1(r2SyncStates.searchType, searchTypeOf(filter.searchType)));
352
+ return (await db.select().from(r2SyncStates).where(and$1(...conds))).map((r) => ({
353
+ userId: String(r.userId),
354
+ siteId: r.siteId === "" ? void 0 : r.siteId,
355
+ table: r.table,
356
+ date: r.date,
357
+ searchType: inferSearchType({ searchType: r.searchType === "" ? void 0 : r.searchType }),
358
+ state: r.state,
359
+ updatedAt: r.updatedAt,
360
+ attempts: r.attempts,
361
+ error: r.error ?? void 0
362
+ }));
363
+ }
364
+ async function setSyncState(scope, state, detail) {
365
+ const updatedAt = detail?.at ?? Date.now();
366
+ const errorText = detail?.error ?? null;
367
+ await db.insert(r2SyncStates).values({
368
+ userId: Number(scope.userId),
369
+ siteId: siteIdOf(scope.siteId),
370
+ table: scope.table,
371
+ date: scope.date,
372
+ searchType: searchTypeOf(scope.searchType),
373
+ state,
374
+ updatedAt,
375
+ attempts: state === "inflight" ? 1 : 0,
376
+ error: errorText
377
+ }).onConflictDoUpdate({
378
+ target: [
379
+ r2SyncStates.userId,
380
+ r2SyncStates.siteId,
381
+ r2SyncStates.table,
382
+ r2SyncStates.date,
383
+ r2SyncStates.searchType
384
+ ],
385
+ set: {
386
+ state: sql$1`excluded.state`,
387
+ updatedAt: sql$1`excluded.updated_at`,
388
+ attempts: sql$1`CASE WHEN excluded.state = 'inflight' THEN attempts + 1 ELSE attempts END`,
389
+ error: sql$1`CASE
390
+ WHEN excluded.state = 'done' THEN NULL
391
+ WHEN excluded.state = 'inflight' THEN error
392
+ ELSE excluded.error
393
+ END`
394
+ }
395
+ }).run();
396
+ }
397
+ async function withLock(scope, fn) {
398
+ const key = lockScopeKey(scope);
399
+ const holderId = crypto.randomUUID();
400
+ const deadline = Date.now() + LOCK_ACQUIRE_TIMEOUT_MS;
401
+ while (true) {
402
+ const now = Date.now();
403
+ const expiresAt = now + LOCK_TTL_MS;
404
+ await db.insert(r2Locks).values({
405
+ scope: key,
406
+ holderId,
407
+ acquiredAt: now,
408
+ expiresAt
409
+ }).onConflictDoUpdate({
410
+ target: r2Locks.scope,
411
+ set: {
412
+ holderId: sql$1`excluded.holder_id`,
413
+ acquiredAt: sql$1`excluded.acquired_at`,
414
+ expiresAt: sql$1`excluded.expires_at`
415
+ },
416
+ setWhere: lt(r2Locks.expiresAt, now)
417
+ }).run();
418
+ if ((await db.select({ holderId: r2Locks.holderId }).from(r2Locks).where(eq$1(r2Locks.scope, key)).get())?.holderId === holderId) break;
419
+ if (Date.now() >= deadline) throw new Error(`withLock: timed out acquiring ${key} after ${LOCK_ACQUIRE_TIMEOUT_MS}ms`);
420
+ await new Promise((resolve) => setTimeout(resolve, jitterDelay()));
421
+ }
422
+ return await fn().finally(async () => {
423
+ await Promise.resolve(db.delete(r2Locks).where(and$1(eq$1(r2Locks.scope, key), eq$1(r2Locks.holderId, holderId))).run()).catch(() => {});
424
+ });
425
+ }
426
+ async function purgeTenant(filter) {
427
+ const userIdNum = Number(filter.userId);
428
+ const entriesCond = filter.siteId !== void 0 ? and$1(eq$1(r2Manifest.userId, userIdNum), eq$1(r2Manifest.siteId, filter.siteId)) : eq$1(r2Manifest.userId, userIdNum);
429
+ const watermarksCond = filter.siteId !== void 0 ? and$1(eq$1(r2Watermarks.userId, userIdNum), eq$1(r2Watermarks.siteId, filter.siteId)) : eq$1(r2Watermarks.userId, userIdNum);
430
+ const syncStatesCond = filter.siteId !== void 0 ? and$1(eq$1(r2SyncStates.userId, userIdNum), eq$1(r2SyncStates.siteId, filter.siteId)) : eq$1(r2SyncStates.userId, userIdNum);
431
+ const [entriesRows, watermarkRows, syncStateRows] = await Promise.all([
432
+ db.select({ n: sql$1`count(*)` }).from(r2Manifest).where(entriesCond).all(),
433
+ db.select({ n: sql$1`count(*)` }).from(r2Watermarks).where(watermarksCond).all(),
434
+ db.select({ n: sql$1`count(*)` }).from(r2SyncStates).where(syncStatesCond).all()
435
+ ]);
436
+ const entriesRemoved = Number(entriesRows[0]?.n ?? 0);
437
+ const watermarksRemoved = Number(watermarkRows[0]?.n ?? 0);
438
+ const syncStatesRemoved = Number(syncStateRows[0]?.n ?? 0);
439
+ await db.batch([
440
+ db.delete(r2Manifest).where(entriesCond),
441
+ db.delete(r2Watermarks).where(watermarksCond),
442
+ db.delete(r2SyncStates).where(syncStatesCond)
443
+ ]);
444
+ return {
445
+ entriesRemoved,
446
+ watermarksRemoved,
447
+ syncStatesRemoved
448
+ };
449
+ }
450
+ return {
451
+ listLive,
452
+ listAll,
453
+ registerVersion: (entry, superseding) => registerVersions([entry], superseding),
454
+ registerVersions,
455
+ listRetired,
456
+ delete: deleteEntries,
457
+ getWatermarks,
458
+ bumpWatermark,
459
+ getSyncStates,
460
+ setSyncState,
461
+ withLock,
462
+ purgeTenant
463
+ };
464
+ }
124
465
  function createSqliteInsightRunner(opts) {
125
466
  const { executor, logger, rowsAsArrays, schema: schemaOverride } = opts;
126
467
  const callback = async (sql, params, method) => {
@@ -138,4 +479,4 @@ function createSqliteInsightRunner(opts) {
138
479
  }) };
139
480
  }
140
481
  const { scopeFor, mergeScope } = createScopedHelpers(schema);
141
- export { aggClicks, aggCtr, aggImpressions, aggPosition, and, compileSqlite, createEngine, createSqliteInsightRunner, createSqliteResolverAdapter, createSqliteResolverAdapterFromExecutor, eq, gsc_countries, gsc_devices, gsc_keywords, gsc_page_keywords, gsc_pages, gte, lte, mergeScope, probeSqliteRegex, resolveWindow, schema, scopeFor, sql, sqliteResolverAdapter };
482
+ export { aggClicks, aggCtr, aggImpressions, aggPosition, and, compileSqlite, createD1ManifestStore, createEngine, createSqliteInsightRunner, createSqliteResolverAdapter, createSqliteResolverAdapterFromExecutor, eq, gsc_countries, gsc_devices, gsc_keywords, gsc_page_keywords, gsc_pages, gte, lte, mergeScope, probeSqliteRegex, r2Locks, r2Manifest, r2ShadowDiffs, r2SyncStates, r2Watermarks, r2WriteErrors, resolveWindow, schema, scopeFor, sql, sqliteResolverAdapter };
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@gscdump/engine-sqlite",
3
3
  "type": "module",
4
- "version": "0.6.3",
4
+ "version": "0.7.1",
5
5
  "description": "SQLite / D1 engine adapter for @gscdump/analysis — typed analytics over sqlite-proxy executors (Cloudflare D1, libsql).",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
@@ -38,9 +38,9 @@
38
38
  "drizzle-orm": "^0.45.2"
39
39
  },
40
40
  "dependencies": {
41
- "@gscdump/engine": "0.6.3",
42
- "gscdump": "0.6.3",
43
- "@gscdump/analysis": "0.6.3"
41
+ "@gscdump/engine": "0.7.1",
42
+ "@gscdump/analysis": "0.7.1",
43
+ "gscdump": "0.7.1"
44
44
  },
45
45
  "devDependencies": {
46
46
  "drizzle-orm": "^0.45.2",