@harperfast/template-react-ts-studio 1.11.7 → 1.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/harper-best-practices/AGENTS.md +56 -80
- package/.agents/skills/harper-best-practices/rules/vector-indexing.md +62 -86
- package/README.md +48 -0
- package/agent/skills/harper-best-practices/AGENTS.md +56 -80
- package/agent/skills/harper-best-practices/rules/vector-indexing.md +62 -86
- package/package.json +1 -1
- package/skills-lock.json +1 -1
- package/.github/workflow/deploy.yaml +0 -33
|
@@ -359,7 +359,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
359
359
|
}
|
|
360
360
|
```
|
|
361
361
|
|
|
362
|
-
2. **Query nearest neighbors** using the `sort` parameter
|
|
362
|
+
2. **Query nearest neighbors** using `Document.search()` with the `sort` parameter. Set `attribute` to the indexed field and `target` to the query vector:
|
|
363
363
|
|
|
364
364
|
```javascript
|
|
365
365
|
let results = Document.search({
|
|
@@ -368,7 +368,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
368
368
|
});
|
|
369
369
|
```
|
|
370
370
|
|
|
371
|
-
3. **Combine with filter conditions** to narrow results before or during traversal:
|
|
371
|
+
3. **Combine with filter conditions** to narrow results before or during graph traversal. Selective conditions are automatically diverted to an exact-scan strategy:
|
|
372
372
|
|
|
373
373
|
```javascript
|
|
374
374
|
let results = Document.search({
|
|
@@ -378,9 +378,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
378
378
|
});
|
|
379
379
|
```
|
|
380
380
|
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
4. **Use a `vectorFilter` function** for predicates not expressible as conditions (JavaScript API only):
|
|
381
|
+
4. **Apply a function predicate during traversal** using `vectorFilter` (JavaScript API only). The function receives each candidate record and must return a synchronous boolean. It must be side-effect free and fast:
|
|
384
382
|
|
|
385
383
|
```javascript
|
|
386
384
|
let results = Document.search(
|
|
@@ -394,9 +392,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
394
392
|
);
|
|
395
393
|
```
|
|
396
394
|
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
5. **Filter by distance threshold** using `target` on a condition instead of `sort`:
|
|
395
|
+
5. **Filter by distance threshold** using `target` directly on a condition alongside `comparator` and `value`. This returns matches within the threshold without using `sort`:
|
|
400
396
|
|
|
401
397
|
```javascript
|
|
402
398
|
let results = Document.search({
|
|
@@ -409,7 +405,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
409
405
|
});
|
|
410
406
|
```
|
|
411
407
|
|
|
412
|
-
6. **Include computed distance in results**
|
|
408
|
+
6. **Include computed distance in results** by adding `$distance` to `select`. Works with both `sort`-based and threshold queries:
|
|
413
409
|
|
|
414
410
|
```javascript
|
|
415
411
|
let results = Document.search({
|
|
@@ -419,9 +415,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
419
415
|
});
|
|
420
416
|
```
|
|
421
417
|
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
7. **Override the distance function or exploration budget per query** via options on the `sort` descriptor:
|
|
418
|
+
7. **Tune per-query search options** on the `sort` descriptor using `distance` and `ef`:
|
|
425
419
|
|
|
426
420
|
```javascript
|
|
427
421
|
let results = Document.search({
|
|
@@ -430,10 +424,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
430
424
|
});
|
|
431
425
|
```
|
|
432
426
|
|
|
433
|
-
|
|
434
|
-
- `ef` — overrides the search exploration budget for this query. Higher values improve recall at the cost of latency.
|
|
435
|
-
|
|
436
|
-
8. **Tune filtered traversal** when a `vectorFilter` is very selective by raising `ef` or `filterExpansion`:
|
|
427
|
+
8. **Tune filtered traversal** with `ef` and `filterExpansion` when a `vectorFilter` is very selective. The visit budget is `ef * filterExpansion` nodes (`filterExpansion` defaults to `24`):
|
|
437
428
|
|
|
438
429
|
```javascript
|
|
439
430
|
let results = Document.search(
|
|
@@ -446,60 +437,48 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
446
437
|
);
|
|
447
438
|
```
|
|
448
439
|
|
|
449
|
-
|
|
440
|
+
9. **Enforce row-level access control** using `rowFilter` on search and subscription targets (JavaScript API only). Attach it in an operation override. For vector queries, `rowFilter` participates in HNSW traversal so callers receive the k nearest _matching_ records:
|
|
450
441
|
|
|
451
|
-
|
|
442
|
+
```javascript
|
|
443
|
+
function canReadReport(record, context) {
|
|
444
|
+
const user = context.user;
|
|
445
|
+
if (user?.role?.permission?.super_user) return true;
|
|
446
|
+
return user?.username != null && record.ownerId != null && record.ownerId === user.username;
|
|
447
|
+
}
|
|
452
448
|
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
449
|
+
export class Reports extends tables.Reports {
|
|
450
|
+
search(target) {
|
|
451
|
+
target.rowFilter = canReadReport;
|
|
452
|
+
return super.search(target);
|
|
453
|
+
}
|
|
458
454
|
}
|
|
459
455
|
```
|
|
460
456
|
|
|
461
|
-
|
|
462
|
-
| ---------------------- | ----------------- | ------------------------------------------------------------------------------------------------ |
|
|
463
|
-
| `distance` | `"cosine"` | Distance function: `"cosine"`, `"euclidean"`, or `"dotProduct"` |
|
|
464
|
-
| `efConstruction` | `100` | Max nodes explored during index construction. Higher = better recall, lower = better performance |
|
|
465
|
-
| `M` | `16` | Preferred connections per graph layer |
|
|
466
|
-
| `optimizeRouting` | `0.5` | Heuristic aggressiveness for omitting redundant connections (0 = off, 1 = most aggressive) |
|
|
467
|
-
| `mL` | computed from `M` | Normalization factor for level generation |
|
|
468
|
-
| `efConstructionSearch` | auto-scaled | Max nodes explored during search. When unset, auto-scales with index size |
|
|
469
|
-
| `quantization` | — | `"int8"` stores vectors quantized to int8 |
|
|
470
|
-
| `filterExpansion` | `24` | Visit-budget multiplier for filtered search |
|
|
457
|
+
##### HNSW Index Parameters
|
|
471
458
|
|
|
472
|
-
|
|
459
|
+
Configure parameters directly on `@indexed(type: "HNSW", ...)`:
|
|
473
460
|
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
461
|
+
| Parameter | Default | Description |
|
|
462
|
+
| ---------------------- | ----------------- | ------------------------------------------------------------------------------------------------ |
|
|
463
|
+
| `distance` | `"cosine"` | Distance function: `"cosine"`, `"euclidean"`, or `"dotProduct"` |
|
|
464
|
+
| `efConstruction` | `100` | Max nodes explored during index construction. Higher = better recall, lower = better performance |
|
|
465
|
+
| `M` | `16` | Preferred connections per graph layer |
|
|
466
|
+
| `optimizeRouting` | `0.5` | Heuristic aggressiveness for omitting redundant connections (0 = off, 1 = most aggressive) |
|
|
467
|
+
| `mL` | computed from `M` | Normalization factor for level generation |
|
|
468
|
+
| `efConstructionSearch` | auto-scaled | Max nodes explored during search. When unset, auto-scales with index size |
|
|
469
|
+
| `quantization` | — | `"int8"` stores vectors quantized to int8 |
|
|
470
|
+
| `filterExpansion` | `24` | Visit-budget multiplier for filtered search: visits at most `ef * filterExpansion` nodes |
|
|
482
471
|
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
11. **Implement record-level access control** by overriding `allowRead(user, target, context)` on the table resource. During vector queries the check participates in graph traversal, so a restricted user receives the k nearest records they are allowed to see:
|
|
486
|
-
|
|
487
|
-
```javascript
|
|
488
|
-
export class Reports extends tables.Reports {
|
|
489
|
-
allowRead(user, target, context) {
|
|
490
|
-
if (!super.allowRead(user, target, context)) return false;
|
|
491
|
-
if (user.role.permission.super_user) return true;
|
|
492
|
-
if (this.ownerId == null) return true;
|
|
493
|
-
return this.ownerId === user.id;
|
|
494
|
-
}
|
|
495
|
-
}
|
|
496
|
-
```
|
|
472
|
+
Per-query `sort` descriptor options:
|
|
497
473
|
|
|
498
|
-
|
|
474
|
+
| Option | Values | Description |
|
|
475
|
+
| ---------- | ----------------------------------------- | ------------------------------------------------------ |
|
|
476
|
+
| `distance` | `"cosine"`, `"euclidean"`, `"dotProduct"` | Overrides the index's distance function for this query |
|
|
477
|
+
| `ef` | integer | Overrides the search exploration budget for this query |
|
|
499
478
|
|
|
500
479
|
#### Examples
|
|
501
480
|
|
|
502
|
-
**
|
|
481
|
+
**Index with custom HNSW parameters:**
|
|
503
482
|
|
|
504
483
|
```graphql
|
|
505
484
|
type Document @table {
|
|
@@ -509,7 +488,16 @@ type Document @table {
|
|
|
509
488
|
}
|
|
510
489
|
```
|
|
511
490
|
|
|
512
|
-
**
|
|
491
|
+
**Index with int8 quantization:**
|
|
492
|
+
|
|
493
|
+
```graphql
|
|
494
|
+
type Document @table {
|
|
495
|
+
id: Long @primaryKey
|
|
496
|
+
textEmbeddings: [Float] @indexed(type: "HNSW", quantization: "int8")
|
|
497
|
+
}
|
|
498
|
+
```
|
|
499
|
+
|
|
500
|
+
**Nearest-neighbor search with distance included:**
|
|
513
501
|
|
|
514
502
|
```javascript
|
|
515
503
|
let results = Document.search({
|
|
@@ -519,7 +507,7 @@ let results = Document.search({
|
|
|
519
507
|
});
|
|
520
508
|
```
|
|
521
509
|
|
|
522
|
-
**Filtered
|
|
510
|
+
**Filtered traversal with tuned budget:**
|
|
523
511
|
|
|
524
512
|
```javascript
|
|
525
513
|
let results = Document.search(
|
|
@@ -532,28 +520,16 @@ let results = Document.search(
|
|
|
532
520
|
);
|
|
533
521
|
```
|
|
534
522
|
|
|
535
|
-
**Distance threshold query (no ranking, cutoff only):**
|
|
536
|
-
|
|
537
|
-
```javascript
|
|
538
|
-
let results = Document.search({
|
|
539
|
-
conditions: {
|
|
540
|
-
attribute: 'textEmbeddings',
|
|
541
|
-
comparator: 'lt',
|
|
542
|
-
value: 0.1,
|
|
543
|
-
target: searchVector,
|
|
544
|
-
},
|
|
545
|
-
});
|
|
546
|
-
```
|
|
547
|
-
|
|
548
523
|
#### Notes
|
|
549
524
|
|
|
550
|
-
-
|
|
551
|
-
-
|
|
552
|
-
- `
|
|
553
|
-
- `
|
|
554
|
-
-
|
|
555
|
-
-
|
|
556
|
-
-
|
|
525
|
+
- `vectorFilter` and `rowFilter` are available from the JavaScript API only; they cannot be set through REST or QUERY request data.
|
|
526
|
+
- `vectorFilter` functions must be synchronous, side-effect free, and fast — they can run once per candidate record visited during traversal; verdicts are memoized per query. Records passed to them are frozen.
|
|
527
|
+
- `rowFilter` does not apply to a direct primary-key `get`.
|
|
528
|
+
- Changing `efConstructionSearch` on an existing index does not trigger a rebuild. Structural parameters (`distance`, `M`, `efConstruction`, `quantization`) do rebuild the index when changed.
|
|
529
|
+
- With `quantization: "int8"`, nearest-neighbor `sort` queries re-rank results against full-precision vectors, restoring exact ordering and exact `$distance` values. Distance-threshold (`lt`/`le`) queries filter on the approximate distance.
|
|
530
|
+
- The correct parameter name is `efConstruction` (seeds the construction budget) and `efConstructionSearch` (controls search budget). The name `efSearchConstruction` is a previous documentation error.
|
|
531
|
+
- When no `ef` is passed and `efConstructionSearch` (or `efConstruction`) is not explicitly set, the search budget auto-scales with index size.
|
|
532
|
+
- `cosine` is the default distance function when `distance` is not specified.
|
|
557
533
|
|
|
558
534
|
### 1.5 Using the Blob Data Type
|
|
559
535
|
|
|
@@ -5,8 +5,8 @@ metadata:
|
|
|
5
5
|
mode: generate
|
|
6
6
|
sources:
|
|
7
7
|
- reference/v5/database/schema.md#Vector Indexing
|
|
8
|
-
sourceCommit:
|
|
9
|
-
inputHash:
|
|
8
|
+
sourceCommit: d4cbc1a7dd400462e4a3243f944b3a75d89b29ca
|
|
9
|
+
inputHash: 1dae788bc850ea90
|
|
10
10
|
---
|
|
11
11
|
|
|
12
12
|
# Vector Indexing
|
|
@@ -28,7 +28,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
28
28
|
}
|
|
29
29
|
```
|
|
30
30
|
|
|
31
|
-
2. **Query nearest neighbors** using the `sort` parameter
|
|
31
|
+
2. **Query nearest neighbors** using `Document.search()` with the `sort` parameter. Set `attribute` to the indexed field and `target` to the query vector:
|
|
32
32
|
|
|
33
33
|
```javascript
|
|
34
34
|
let results = Document.search({
|
|
@@ -37,7 +37,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
37
37
|
});
|
|
38
38
|
```
|
|
39
39
|
|
|
40
|
-
3. **Combine with filter conditions** to narrow results before or during traversal:
|
|
40
|
+
3. **Combine with filter conditions** to narrow results before or during graph traversal. Selective conditions are automatically diverted to an exact-scan strategy:
|
|
41
41
|
|
|
42
42
|
```javascript
|
|
43
43
|
let results = Document.search({
|
|
@@ -47,9 +47,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
47
47
|
});
|
|
48
48
|
```
|
|
49
49
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
4. **Use a `vectorFilter` function** for predicates not expressible as conditions (JavaScript API only):
|
|
50
|
+
4. **Apply a function predicate during traversal** using `vectorFilter` (JavaScript API only). The function receives each candidate record and must return a synchronous boolean. It must be side-effect free and fast:
|
|
53
51
|
|
|
54
52
|
```javascript
|
|
55
53
|
let results = Document.search(
|
|
@@ -63,9 +61,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
63
61
|
);
|
|
64
62
|
```
|
|
65
63
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
5. **Filter by distance threshold** using `target` on a condition instead of `sort`:
|
|
64
|
+
5. **Filter by distance threshold** using `target` directly on a condition alongside `comparator` and `value`. This returns matches within the threshold without using `sort`:
|
|
69
65
|
|
|
70
66
|
```javascript
|
|
71
67
|
let results = Document.search({
|
|
@@ -78,7 +74,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
78
74
|
});
|
|
79
75
|
```
|
|
80
76
|
|
|
81
|
-
6. **Include computed distance in results**
|
|
77
|
+
6. **Include computed distance in results** by adding `$distance` to `select`. Works with both `sort`-based and threshold queries:
|
|
82
78
|
|
|
83
79
|
```javascript
|
|
84
80
|
let results = Document.search({
|
|
@@ -88,9 +84,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
88
84
|
});
|
|
89
85
|
```
|
|
90
86
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
7. **Override the distance function or exploration budget per query** via options on the `sort` descriptor:
|
|
87
|
+
7. **Tune per-query search options** on the `sort` descriptor using `distance` and `ef`:
|
|
94
88
|
|
|
95
89
|
```javascript
|
|
96
90
|
let results = Document.search({
|
|
@@ -99,10 +93,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
99
93
|
});
|
|
100
94
|
```
|
|
101
95
|
|
|
102
|
-
|
|
103
|
-
- `ef` — overrides the search exploration budget for this query. Higher values improve recall at the cost of latency.
|
|
104
|
-
|
|
105
|
-
8. **Tune filtered traversal** when a `vectorFilter` is very selective by raising `ef` or `filterExpansion`:
|
|
96
|
+
8. **Tune filtered traversal** with `ef` and `filterExpansion` when a `vectorFilter` is very selective. The visit budget is `ef * filterExpansion` nodes (`filterExpansion` defaults to `24`):
|
|
106
97
|
|
|
107
98
|
```javascript
|
|
108
99
|
let results = Document.search(
|
|
@@ -115,60 +106,48 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
115
106
|
);
|
|
116
107
|
```
|
|
117
108
|
|
|
118
|
-
|
|
109
|
+
9. **Enforce row-level access control** using `rowFilter` on search and subscription targets (JavaScript API only). Attach it in an operation override. For vector queries, `rowFilter` participates in HNSW traversal so callers receive the k nearest _matching_ records:
|
|
119
110
|
|
|
120
|
-
|
|
111
|
+
```javascript
|
|
112
|
+
function canReadReport(record, context) {
|
|
113
|
+
const user = context.user;
|
|
114
|
+
if (user?.role?.permission?.super_user) return true;
|
|
115
|
+
return user?.username != null && record.ownerId != null && record.ownerId === user.username;
|
|
116
|
+
}
|
|
121
117
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
118
|
+
export class Reports extends tables.Reports {
|
|
119
|
+
search(target) {
|
|
120
|
+
target.rowFilter = canReadReport;
|
|
121
|
+
return super.search(target);
|
|
122
|
+
}
|
|
127
123
|
}
|
|
128
124
|
```
|
|
129
125
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
Graph navigation uses quantized distances. For `sort` queries, Harper re-ranks results against full-precision vectors, restoring exact ordering and exact `$distance` values. Distance-threshold queries filter on the approximate distance.
|
|
153
|
-
|
|
154
|
-
11. **Implement record-level access control** by overriding `allowRead(user, target, context)` on the table resource. During vector queries the check participates in graph traversal, so a restricted user receives the k nearest records they are allowed to see:
|
|
155
|
-
|
|
156
|
-
```javascript
|
|
157
|
-
export class Reports extends tables.Reports {
|
|
158
|
-
allowRead(user, target, context) {
|
|
159
|
-
if (!super.allowRead(user, target, context)) return false;
|
|
160
|
-
if (user.role.permission.super_user) return true;
|
|
161
|
-
if (this.ownerId == null) return true;
|
|
162
|
-
return this.ownerId === user.id;
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
```
|
|
166
|
-
|
|
167
|
-
The check must be synchronous, side-effect free, and fast. `this` is the frozen record during per-record evaluation. A thrown exception denies that record (fail closed).
|
|
126
|
+
### HNSW Index Parameters
|
|
127
|
+
|
|
128
|
+
Configure parameters directly on `@indexed(type: "HNSW", ...)`:
|
|
129
|
+
|
|
130
|
+
| Parameter | Default | Description |
|
|
131
|
+
| ---------------------- | ----------------- | ------------------------------------------------------------------------------------------------ |
|
|
132
|
+
| `distance` | `"cosine"` | Distance function: `"cosine"`, `"euclidean"`, or `"dotProduct"` |
|
|
133
|
+
| `efConstruction` | `100` | Max nodes explored during index construction. Higher = better recall, lower = better performance |
|
|
134
|
+
| `M` | `16` | Preferred connections per graph layer |
|
|
135
|
+
| `optimizeRouting` | `0.5` | Heuristic aggressiveness for omitting redundant connections (0 = off, 1 = most aggressive) |
|
|
136
|
+
| `mL` | computed from `M` | Normalization factor for level generation |
|
|
137
|
+
| `efConstructionSearch` | auto-scaled | Max nodes explored during search. When unset, auto-scales with index size |
|
|
138
|
+
| `quantization` | — | `"int8"` stores vectors quantized to int8 |
|
|
139
|
+
| `filterExpansion` | `24` | Visit-budget multiplier for filtered search: visits at most `ef * filterExpansion` nodes |
|
|
140
|
+
|
|
141
|
+
Per-query `sort` descriptor options:
|
|
142
|
+
|
|
143
|
+
| Option | Values | Description |
|
|
144
|
+
| ---------- | ----------------------------------------- | ------------------------------------------------------ |
|
|
145
|
+
| `distance` | `"cosine"`, `"euclidean"`, `"dotProduct"` | Overrides the index's distance function for this query |
|
|
146
|
+
| `ef` | integer | Overrides the search exploration budget for this query |
|
|
168
147
|
|
|
169
148
|
## Examples
|
|
170
149
|
|
|
171
|
-
**
|
|
150
|
+
**Index with custom HNSW parameters:**
|
|
172
151
|
|
|
173
152
|
```graphql
|
|
174
153
|
type Document @table {
|
|
@@ -178,7 +157,16 @@ type Document @table {
|
|
|
178
157
|
}
|
|
179
158
|
```
|
|
180
159
|
|
|
181
|
-
**
|
|
160
|
+
**Index with int8 quantization:**
|
|
161
|
+
|
|
162
|
+
```graphql
|
|
163
|
+
type Document @table {
|
|
164
|
+
id: Long @primaryKey
|
|
165
|
+
textEmbeddings: [Float] @indexed(type: "HNSW", quantization: "int8")
|
|
166
|
+
}
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
**Nearest-neighbor search with distance included:**
|
|
182
170
|
|
|
183
171
|
```javascript
|
|
184
172
|
let results = Document.search({
|
|
@@ -188,7 +176,7 @@ let results = Document.search({
|
|
|
188
176
|
});
|
|
189
177
|
```
|
|
190
178
|
|
|
191
|
-
**Filtered
|
|
179
|
+
**Filtered traversal with tuned budget:**
|
|
192
180
|
|
|
193
181
|
```javascript
|
|
194
182
|
let results = Document.search(
|
|
@@ -201,25 +189,13 @@ let results = Document.search(
|
|
|
201
189
|
);
|
|
202
190
|
```
|
|
203
191
|
|
|
204
|
-
**Distance threshold query (no ranking, cutoff only):**
|
|
205
|
-
|
|
206
|
-
```javascript
|
|
207
|
-
let results = Document.search({
|
|
208
|
-
conditions: {
|
|
209
|
-
attribute: 'textEmbeddings',
|
|
210
|
-
comparator: 'lt',
|
|
211
|
-
value: 0.1,
|
|
212
|
-
target: searchVector,
|
|
213
|
-
},
|
|
214
|
-
});
|
|
215
|
-
```
|
|
216
|
-
|
|
217
192
|
## Notes
|
|
218
193
|
|
|
219
|
-
-
|
|
220
|
-
-
|
|
221
|
-
- `
|
|
222
|
-
- `
|
|
223
|
-
-
|
|
224
|
-
-
|
|
225
|
-
-
|
|
194
|
+
- `vectorFilter` and `rowFilter` are available from the JavaScript API only; they cannot be set through REST or QUERY request data.
|
|
195
|
+
- `vectorFilter` functions must be synchronous, side-effect free, and fast — they can run once per candidate record visited during traversal; verdicts are memoized per query. Records passed to them are frozen.
|
|
196
|
+
- `rowFilter` does not apply to a direct primary-key `get`.
|
|
197
|
+
- Changing `efConstructionSearch` on an existing index does not trigger a rebuild. Structural parameters (`distance`, `M`, `efConstruction`, `quantization`) do rebuild the index when changed.
|
|
198
|
+
- With `quantization: "int8"`, nearest-neighbor `sort` queries re-rank results against full-precision vectors, restoring exact ordering and exact `$distance` values. Distance-threshold (`lt`/`le`) queries filter on the approximate distance.
|
|
199
|
+
- The correct parameter name is `efConstruction` (seeds the construction budget) and `efConstructionSearch` (controls search budget). The name `efSearchConstruction` is a previous documentation error.
|
|
200
|
+
- When no `ef` is passed and `efConstructionSearch` (or `efConstruction`) is not explicitly set, the search budget auto-scales with index size.
|
|
201
|
+
- `cosine` is the default distance function when `distance` is not specified.
|
package/README.md
CHANGED
|
@@ -96,6 +96,54 @@ You should see the following:
|
|
|
96
96
|
|
|
97
97
|
Take a look at the [default configuration](./config.yaml), which specifies how files are handled in your application.
|
|
98
98
|
|
|
99
|
+
### One-time setup (private repos)
|
|
100
|
+
|
|
101
|
+
So the cluster can clone your private repository, give it a read-only token — sealed on your machine, stored encrypted:
|
|
102
|
+
|
|
103
|
+
```sh
|
|
104
|
+
npm run deploy:setup
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
This fetches your cluster's public key, has you provide a GitHub token (a fine-grained PAT with **Contents: Read-only**, or your `gh` CLI session), **encrypts it locally**, and stores only the ciphertext in the cluster's secret store. The plaintext never leaves your machine; the cluster decrypts it in memory only while cloning. Because the token is durable, rollbacks keep working for as long as it's valid.
|
|
108
|
+
|
|
109
|
+
> Public repo? Skip this step and drop `credential=true` from the `deploy` script — no credential is needed.
|
|
110
|
+
|
|
111
|
+
### Deploy
|
|
112
|
+
|
|
113
|
+
```sh
|
|
114
|
+
npm run deploy
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
This deploys the current commit over `git+https` — commit and push first, since the cluster clones from GitHub and only sees pushed commits. To roll back, check out an older commit and run it again.
|
|
118
|
+
|
|
119
|
+
### Deploy automatically from CI
|
|
120
|
+
|
|
121
|
+
The included [GitHub Actions workflow](./.github/workflows/deploy.yaml) deploys whenever you push a version tag:
|
|
122
|
+
|
|
123
|
+
```sh
|
|
124
|
+
git tag v1.0.0
|
|
125
|
+
git push --tags
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Add these repository secrets first, under **Settings → Secrets and variables → Actions**:
|
|
129
|
+
|
|
130
|
+
- `HARPER_CLI_TARGET` — your cluster's operations URL (e.g. `https://your-cluster.harperdb.io:9925`)
|
|
131
|
+
- `HARPER_CLI_REFRESH_TOKEN` — a long-lived token CI authenticates with, so no password is stored
|
|
132
|
+
|
|
133
|
+
Set both in one command — this pipes the credentials straight from your cluster into GitHub, so the token never appears on screen or in your shell history:
|
|
134
|
+
|
|
135
|
+
```sh
|
|
136
|
+
harper login --for-ci | gh secret set --env-file -
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
(No [`gh` CLI](https://cli.github.com)? `harper login --for-ci | pbcopy` copies the two lines for you to paste in by hand.)
|
|
140
|
+
|
|
141
|
+
The clone credential already lives in the cluster from `npm run deploy:setup`, so CI never handles a token itself.
|
|
142
|
+
|
|
143
|
+
### Private npm dependencies
|
|
144
|
+
|
|
145
|
+
If your app depends on private npm packages, run `npm run deploy:setup` again and choose the npm registry — the same sealed-token flow, stored as a separate credential.
|
|
146
|
+
|
|
99
147
|
## Keep Going!
|
|
100
148
|
|
|
101
149
|
For more information about getting started with Harper and building applications, see our [getting started guide](https://docs.harperdb.io/docs).
|
|
@@ -359,7 +359,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
359
359
|
}
|
|
360
360
|
```
|
|
361
361
|
|
|
362
|
-
2. **Query nearest neighbors** using the `sort` parameter
|
|
362
|
+
2. **Query nearest neighbors** using `Document.search()` with the `sort` parameter. Set `attribute` to the indexed field and `target` to the query vector:
|
|
363
363
|
|
|
364
364
|
```javascript
|
|
365
365
|
let results = Document.search({
|
|
@@ -368,7 +368,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
368
368
|
});
|
|
369
369
|
```
|
|
370
370
|
|
|
371
|
-
3. **Combine with filter conditions** to narrow results before or during traversal:
|
|
371
|
+
3. **Combine with filter conditions** to narrow results before or during graph traversal. Selective conditions are automatically diverted to an exact-scan strategy:
|
|
372
372
|
|
|
373
373
|
```javascript
|
|
374
374
|
let results = Document.search({
|
|
@@ -378,9 +378,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
378
378
|
});
|
|
379
379
|
```
|
|
380
380
|
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
4. **Use a `vectorFilter` function** for predicates not expressible as conditions (JavaScript API only):
|
|
381
|
+
4. **Apply a function predicate during traversal** using `vectorFilter` (JavaScript API only). The function receives each candidate record and must return a synchronous boolean. It must be side-effect free and fast:
|
|
384
382
|
|
|
385
383
|
```javascript
|
|
386
384
|
let results = Document.search(
|
|
@@ -394,9 +392,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
394
392
|
);
|
|
395
393
|
```
|
|
396
394
|
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
5. **Filter by distance threshold** using `target` on a condition instead of `sort`:
|
|
395
|
+
5. **Filter by distance threshold** using `target` directly on a condition alongside `comparator` and `value`. This returns matches within the threshold without using `sort`:
|
|
400
396
|
|
|
401
397
|
```javascript
|
|
402
398
|
let results = Document.search({
|
|
@@ -409,7 +405,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
409
405
|
});
|
|
410
406
|
```
|
|
411
407
|
|
|
412
|
-
6. **Include computed distance in results**
|
|
408
|
+
6. **Include computed distance in results** by adding `$distance` to `select`. Works with both `sort`-based and threshold queries:
|
|
413
409
|
|
|
414
410
|
```javascript
|
|
415
411
|
let results = Document.search({
|
|
@@ -419,9 +415,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
419
415
|
});
|
|
420
416
|
```
|
|
421
417
|
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
7. **Override the distance function or exploration budget per query** via options on the `sort` descriptor:
|
|
418
|
+
7. **Tune per-query search options** on the `sort` descriptor using `distance` and `ef`:
|
|
425
419
|
|
|
426
420
|
```javascript
|
|
427
421
|
let results = Document.search({
|
|
@@ -430,10 +424,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
430
424
|
});
|
|
431
425
|
```
|
|
432
426
|
|
|
433
|
-
|
|
434
|
-
- `ef` — overrides the search exploration budget for this query. Higher values improve recall at the cost of latency.
|
|
435
|
-
|
|
436
|
-
8. **Tune filtered traversal** when a `vectorFilter` is very selective by raising `ef` or `filterExpansion`:
|
|
427
|
+
8. **Tune filtered traversal** with `ef` and `filterExpansion` when a `vectorFilter` is very selective. The visit budget is `ef * filterExpansion` nodes (`filterExpansion` defaults to `24`):
|
|
437
428
|
|
|
438
429
|
```javascript
|
|
439
430
|
let results = Document.search(
|
|
@@ -446,60 +437,48 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
446
437
|
);
|
|
447
438
|
```
|
|
448
439
|
|
|
449
|
-
|
|
440
|
+
9. **Enforce row-level access control** using `rowFilter` on search and subscription targets (JavaScript API only). Attach it in an operation override. For vector queries, `rowFilter` participates in HNSW traversal so callers receive the k nearest _matching_ records:
|
|
450
441
|
|
|
451
|
-
|
|
442
|
+
```javascript
|
|
443
|
+
function canReadReport(record, context) {
|
|
444
|
+
const user = context.user;
|
|
445
|
+
if (user?.role?.permission?.super_user) return true;
|
|
446
|
+
return user?.username != null && record.ownerId != null && record.ownerId === user.username;
|
|
447
|
+
}
|
|
452
448
|
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
449
|
+
export class Reports extends tables.Reports {
|
|
450
|
+
search(target) {
|
|
451
|
+
target.rowFilter = canReadReport;
|
|
452
|
+
return super.search(target);
|
|
453
|
+
}
|
|
458
454
|
}
|
|
459
455
|
```
|
|
460
456
|
|
|
461
|
-
|
|
462
|
-
| ---------------------- | ----------------- | ------------------------------------------------------------------------------------------------ |
|
|
463
|
-
| `distance` | `"cosine"` | Distance function: `"cosine"`, `"euclidean"`, or `"dotProduct"` |
|
|
464
|
-
| `efConstruction` | `100` | Max nodes explored during index construction. Higher = better recall, lower = better performance |
|
|
465
|
-
| `M` | `16` | Preferred connections per graph layer |
|
|
466
|
-
| `optimizeRouting` | `0.5` | Heuristic aggressiveness for omitting redundant connections (0 = off, 1 = most aggressive) |
|
|
467
|
-
| `mL` | computed from `M` | Normalization factor for level generation |
|
|
468
|
-
| `efConstructionSearch` | auto-scaled | Max nodes explored during search. When unset, auto-scales with index size |
|
|
469
|
-
| `quantization` | — | `"int8"` stores vectors quantized to int8 |
|
|
470
|
-
| `filterExpansion` | `24` | Visit-budget multiplier for filtered search |
|
|
457
|
+
##### HNSW Index Parameters
|
|
471
458
|
|
|
472
|
-
|
|
459
|
+
Configure parameters directly on `@indexed(type: "HNSW", ...)`:
|
|
473
460
|
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
461
|
+
| Parameter | Default | Description |
|
|
462
|
+
| ---------------------- | ----------------- | ------------------------------------------------------------------------------------------------ |
|
|
463
|
+
| `distance` | `"cosine"` | Distance function: `"cosine"`, `"euclidean"`, or `"dotProduct"` |
|
|
464
|
+
| `efConstruction` | `100` | Max nodes explored during index construction. Higher = better recall, lower = better performance |
|
|
465
|
+
| `M` | `16` | Preferred connections per graph layer |
|
|
466
|
+
| `optimizeRouting` | `0.5` | Heuristic aggressiveness for omitting redundant connections (0 = off, 1 = most aggressive) |
|
|
467
|
+
| `mL` | computed from `M` | Normalization factor for level generation |
|
|
468
|
+
| `efConstructionSearch` | auto-scaled | Max nodes explored during search. When unset, auto-scales with index size |
|
|
469
|
+
| `quantization` | — | `"int8"` stores vectors quantized to int8 |
|
|
470
|
+
| `filterExpansion` | `24` | Visit-budget multiplier for filtered search: visits at most `ef * filterExpansion` nodes |
|
|
482
471
|
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
11. **Implement record-level access control** by overriding `allowRead(user, target, context)` on the table resource. During vector queries the check participates in graph traversal, so a restricted user receives the k nearest records they are allowed to see:
|
|
486
|
-
|
|
487
|
-
```javascript
|
|
488
|
-
export class Reports extends tables.Reports {
|
|
489
|
-
allowRead(user, target, context) {
|
|
490
|
-
if (!super.allowRead(user, target, context)) return false;
|
|
491
|
-
if (user.role.permission.super_user) return true;
|
|
492
|
-
if (this.ownerId == null) return true;
|
|
493
|
-
return this.ownerId === user.id;
|
|
494
|
-
}
|
|
495
|
-
}
|
|
496
|
-
```
|
|
472
|
+
Per-query `sort` descriptor options:
|
|
497
473
|
|
|
498
|
-
|
|
474
|
+
| Option | Values | Description |
|
|
475
|
+
| ---------- | ----------------------------------------- | ------------------------------------------------------ |
|
|
476
|
+
| `distance` | `"cosine"`, `"euclidean"`, `"dotProduct"` | Overrides the index's distance function for this query |
|
|
477
|
+
| `ef` | integer | Overrides the search exploration budget for this query |
|
|
499
478
|
|
|
500
479
|
#### Examples
|
|
501
480
|
|
|
502
|
-
**
|
|
481
|
+
**Index with custom HNSW parameters:**
|
|
503
482
|
|
|
504
483
|
```graphql
|
|
505
484
|
type Document @table {
|
|
@@ -509,7 +488,16 @@ type Document @table {
|
|
|
509
488
|
}
|
|
510
489
|
```
|
|
511
490
|
|
|
512
|
-
**
|
|
491
|
+
**Index with int8 quantization:**
|
|
492
|
+
|
|
493
|
+
```graphql
|
|
494
|
+
type Document @table {
|
|
495
|
+
id: Long @primaryKey
|
|
496
|
+
textEmbeddings: [Float] @indexed(type: "HNSW", quantization: "int8")
|
|
497
|
+
}
|
|
498
|
+
```
|
|
499
|
+
|
|
500
|
+
**Nearest-neighbor search with distance included:**
|
|
513
501
|
|
|
514
502
|
```javascript
|
|
515
503
|
let results = Document.search({
|
|
@@ -519,7 +507,7 @@ let results = Document.search({
|
|
|
519
507
|
});
|
|
520
508
|
```
|
|
521
509
|
|
|
522
|
-
**Filtered
|
|
510
|
+
**Filtered traversal with tuned budget:**
|
|
523
511
|
|
|
524
512
|
```javascript
|
|
525
513
|
let results = Document.search(
|
|
@@ -532,28 +520,16 @@ let results = Document.search(
|
|
|
532
520
|
);
|
|
533
521
|
```
|
|
534
522
|
|
|
535
|
-
**Distance threshold query (no ranking, cutoff only):**
|
|
536
|
-
|
|
537
|
-
```javascript
|
|
538
|
-
let results = Document.search({
|
|
539
|
-
conditions: {
|
|
540
|
-
attribute: 'textEmbeddings',
|
|
541
|
-
comparator: 'lt',
|
|
542
|
-
value: 0.1,
|
|
543
|
-
target: searchVector,
|
|
544
|
-
},
|
|
545
|
-
});
|
|
546
|
-
```
|
|
547
|
-
|
|
548
523
|
#### Notes
|
|
549
524
|
|
|
550
|
-
-
|
|
551
|
-
-
|
|
552
|
-
- `
|
|
553
|
-
- `
|
|
554
|
-
-
|
|
555
|
-
-
|
|
556
|
-
-
|
|
525
|
+
- `vectorFilter` and `rowFilter` are available from the JavaScript API only; they cannot be set through REST or QUERY request data.
|
|
526
|
+
- `vectorFilter` functions must be synchronous, side-effect free, and fast — they can run once per candidate record visited during traversal; verdicts are memoized per query. Records passed to them are frozen.
|
|
527
|
+
- `rowFilter` does not apply to a direct primary-key `get`.
|
|
528
|
+
- Changing `efConstructionSearch` on an existing index does not trigger a rebuild. Structural parameters (`distance`, `M`, `efConstruction`, `quantization`) do rebuild the index when changed.
|
|
529
|
+
- With `quantization: "int8"`, nearest-neighbor `sort` queries re-rank results against full-precision vectors, restoring exact ordering and exact `$distance` values. Distance-threshold (`lt`/`le`) queries filter on the approximate distance.
|
|
530
|
+
- The correct parameter name is `efConstruction` (seeds the construction budget) and `efConstructionSearch` (controls search budget). The name `efSearchConstruction` is a previous documentation error.
|
|
531
|
+
- When no `ef` is passed and `efConstructionSearch` (or `efConstruction`) is not explicitly set, the search budget auto-scales with index size.
|
|
532
|
+
- `cosine` is the default distance function when `distance` is not specified.
|
|
557
533
|
|
|
558
534
|
### 1.5 Using the Blob Data Type
|
|
559
535
|
|
|
@@ -5,8 +5,8 @@ metadata:
|
|
|
5
5
|
mode: generate
|
|
6
6
|
sources:
|
|
7
7
|
- reference/v5/database/schema.md#Vector Indexing
|
|
8
|
-
sourceCommit:
|
|
9
|
-
inputHash:
|
|
8
|
+
sourceCommit: d4cbc1a7dd400462e4a3243f944b3a75d89b29ca
|
|
9
|
+
inputHash: 1dae788bc850ea90
|
|
10
10
|
---
|
|
11
11
|
|
|
12
12
|
# Vector Indexing
|
|
@@ -28,7 +28,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
28
28
|
}
|
|
29
29
|
```
|
|
30
30
|
|
|
31
|
-
2. **Query nearest neighbors** using the `sort` parameter
|
|
31
|
+
2. **Query nearest neighbors** using `Document.search()` with the `sort` parameter. Set `attribute` to the indexed field and `target` to the query vector:
|
|
32
32
|
|
|
33
33
|
```javascript
|
|
34
34
|
let results = Document.search({
|
|
@@ -37,7 +37,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
37
37
|
});
|
|
38
38
|
```
|
|
39
39
|
|
|
40
|
-
3. **Combine with filter conditions** to narrow results before or during traversal:
|
|
40
|
+
3. **Combine with filter conditions** to narrow results before or during graph traversal. Selective conditions are automatically diverted to an exact-scan strategy:
|
|
41
41
|
|
|
42
42
|
```javascript
|
|
43
43
|
let results = Document.search({
|
|
@@ -47,9 +47,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
47
47
|
});
|
|
48
48
|
```
|
|
49
49
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
4. **Use a `vectorFilter` function** for predicates not expressible as conditions (JavaScript API only):
|
|
50
|
+
4. **Apply a function predicate during traversal** using `vectorFilter` (JavaScript API only). The function receives each candidate record and must return a synchronous boolean. It must be side-effect free and fast:
|
|
53
51
|
|
|
54
52
|
```javascript
|
|
55
53
|
let results = Document.search(
|
|
@@ -63,9 +61,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
63
61
|
);
|
|
64
62
|
```
|
|
65
63
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
5. **Filter by distance threshold** using `target` on a condition instead of `sort`:
|
|
64
|
+
5. **Filter by distance threshold** using `target` directly on a condition alongside `comparator` and `value`. This returns matches within the threshold without using `sort`:
|
|
69
65
|
|
|
70
66
|
```javascript
|
|
71
67
|
let results = Document.search({
|
|
@@ -78,7 +74,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
78
74
|
});
|
|
79
75
|
```
|
|
80
76
|
|
|
81
|
-
6. **Include computed distance in results**
|
|
77
|
+
6. **Include computed distance in results** by adding `$distance` to `select`. Works with both `sort`-based and threshold queries:
|
|
82
78
|
|
|
83
79
|
```javascript
|
|
84
80
|
let results = Document.search({
|
|
@@ -88,9 +84,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
88
84
|
});
|
|
89
85
|
```
|
|
90
86
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
7. **Override the distance function or exploration budget per query** via options on the `sort` descriptor:
|
|
87
|
+
7. **Tune per-query search options** on the `sort` descriptor using `distance` and `ef`:
|
|
94
88
|
|
|
95
89
|
```javascript
|
|
96
90
|
let results = Document.search({
|
|
@@ -99,10 +93,7 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
99
93
|
});
|
|
100
94
|
```
|
|
101
95
|
|
|
102
|
-
|
|
103
|
-
- `ef` — overrides the search exploration budget for this query. Higher values improve recall at the cost of latency.
|
|
104
|
-
|
|
105
|
-
8. **Tune filtered traversal** when a `vectorFilter` is very selective by raising `ef` or `filterExpansion`:
|
|
96
|
+
8. **Tune filtered traversal** with `ef` and `filterExpansion` when a `vectorFilter` is very selective. The visit budget is `ef * filterExpansion` nodes (`filterExpansion` defaults to `24`):
|
|
106
97
|
|
|
107
98
|
```javascript
|
|
108
99
|
let results = Document.search(
|
|
@@ -115,60 +106,48 @@ Apply this rule when adding a vector similarity search capability to a Harper ta
|
|
|
115
106
|
);
|
|
116
107
|
```
|
|
117
108
|
|
|
118
|
-
|
|
109
|
+
9. **Enforce row-level access control** using `rowFilter` on search and subscription targets (JavaScript API only). Attach it in an operation override. For vector queries, `rowFilter` participates in HNSW traversal so callers receive the k nearest _matching_ records:
|
|
119
110
|
|
|
120
|
-
|
|
111
|
+
```javascript
|
|
112
|
+
function canReadReport(record, context) {
|
|
113
|
+
const user = context.user;
|
|
114
|
+
if (user?.role?.permission?.super_user) return true;
|
|
115
|
+
return user?.username != null && record.ownerId != null && record.ownerId === user.username;
|
|
116
|
+
}
|
|
121
117
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
118
|
+
export class Reports extends tables.Reports {
|
|
119
|
+
search(target) {
|
|
120
|
+
target.rowFilter = canReadReport;
|
|
121
|
+
return super.search(target);
|
|
122
|
+
}
|
|
127
123
|
}
|
|
128
124
|
```
|
|
129
125
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
Graph navigation uses quantized distances. For `sort` queries, Harper re-ranks results against full-precision vectors, restoring exact ordering and exact `$distance` values. Distance-threshold queries filter on the approximate distance.
|
|
153
|
-
|
|
154
|
-
11. **Implement record-level access control** by overriding `allowRead(user, target, context)` on the table resource. During vector queries the check participates in graph traversal, so a restricted user receives the k nearest records they are allowed to see:
|
|
155
|
-
|
|
156
|
-
```javascript
|
|
157
|
-
export class Reports extends tables.Reports {
|
|
158
|
-
allowRead(user, target, context) {
|
|
159
|
-
if (!super.allowRead(user, target, context)) return false;
|
|
160
|
-
if (user.role.permission.super_user) return true;
|
|
161
|
-
if (this.ownerId == null) return true;
|
|
162
|
-
return this.ownerId === user.id;
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
```
|
|
166
|
-
|
|
167
|
-
The check must be synchronous, side-effect free, and fast. `this` is the frozen record during per-record evaluation. A thrown exception denies that record (fail closed).
|
|
126
|
+
### HNSW Index Parameters
|
|
127
|
+
|
|
128
|
+
Configure parameters directly on `@indexed(type: "HNSW", ...)`:
|
|
129
|
+
|
|
130
|
+
| Parameter | Default | Description |
|
|
131
|
+
| ---------------------- | ----------------- | ------------------------------------------------------------------------------------------------ |
|
|
132
|
+
| `distance` | `"cosine"` | Distance function: `"cosine"`, `"euclidean"`, or `"dotProduct"` |
|
|
133
|
+
| `efConstruction` | `100` | Max nodes explored during index construction. Higher = better recall, lower = better performance |
|
|
134
|
+
| `M` | `16` | Preferred connections per graph layer |
|
|
135
|
+
| `optimizeRouting` | `0.5` | Heuristic aggressiveness for omitting redundant connections (0 = off, 1 = most aggressive) |
|
|
136
|
+
| `mL` | computed from `M` | Normalization factor for level generation |
|
|
137
|
+
| `efConstructionSearch` | auto-scaled | Max nodes explored during search. When unset, auto-scales with index size |
|
|
138
|
+
| `quantization` | — | `"int8"` stores vectors quantized to int8 |
|
|
139
|
+
| `filterExpansion` | `24` | Visit-budget multiplier for filtered search: visits at most `ef * filterExpansion` nodes |
|
|
140
|
+
|
|
141
|
+
Per-query `sort` descriptor options:
|
|
142
|
+
|
|
143
|
+
| Option | Values | Description |
|
|
144
|
+
| ---------- | ----------------------------------------- | ------------------------------------------------------ |
|
|
145
|
+
| `distance` | `"cosine"`, `"euclidean"`, `"dotProduct"` | Overrides the index's distance function for this query |
|
|
146
|
+
| `ef` | integer | Overrides the search exploration budget for this query |
|
|
168
147
|
|
|
169
148
|
## Examples
|
|
170
149
|
|
|
171
|
-
**
|
|
150
|
+
**Index with custom HNSW parameters:**
|
|
172
151
|
|
|
173
152
|
```graphql
|
|
174
153
|
type Document @table {
|
|
@@ -178,7 +157,16 @@ type Document @table {
|
|
|
178
157
|
}
|
|
179
158
|
```
|
|
180
159
|
|
|
181
|
-
**
|
|
160
|
+
**Index with int8 quantization:**
|
|
161
|
+
|
|
162
|
+
```graphql
|
|
163
|
+
type Document @table {
|
|
164
|
+
id: Long @primaryKey
|
|
165
|
+
textEmbeddings: [Float] @indexed(type: "HNSW", quantization: "int8")
|
|
166
|
+
}
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
**Nearest-neighbor search with distance included:**
|
|
182
170
|
|
|
183
171
|
```javascript
|
|
184
172
|
let results = Document.search({
|
|
@@ -188,7 +176,7 @@ let results = Document.search({
|
|
|
188
176
|
});
|
|
189
177
|
```
|
|
190
178
|
|
|
191
|
-
**Filtered
|
|
179
|
+
**Filtered traversal with tuned budget:**
|
|
192
180
|
|
|
193
181
|
```javascript
|
|
194
182
|
let results = Document.search(
|
|
@@ -201,25 +189,13 @@ let results = Document.search(
|
|
|
201
189
|
);
|
|
202
190
|
```
|
|
203
191
|
|
|
204
|
-
**Distance threshold query (no ranking, cutoff only):**
|
|
205
|
-
|
|
206
|
-
```javascript
|
|
207
|
-
let results = Document.search({
|
|
208
|
-
conditions: {
|
|
209
|
-
attribute: 'textEmbeddings',
|
|
210
|
-
comparator: 'lt',
|
|
211
|
-
value: 0.1,
|
|
212
|
-
target: searchVector,
|
|
213
|
-
},
|
|
214
|
-
});
|
|
215
|
-
```
|
|
216
|
-
|
|
217
192
|
## Notes
|
|
218
193
|
|
|
219
|
-
-
|
|
220
|
-
-
|
|
221
|
-
- `
|
|
222
|
-
- `
|
|
223
|
-
-
|
|
224
|
-
-
|
|
225
|
-
-
|
|
194
|
+
- `vectorFilter` and `rowFilter` are available from the JavaScript API only; they cannot be set through REST or QUERY request data.
|
|
195
|
+
- `vectorFilter` functions must be synchronous, side-effect free, and fast — they can run once per candidate record visited during traversal; verdicts are memoized per query. Records passed to them are frozen.
|
|
196
|
+
- `rowFilter` does not apply to a direct primary-key `get`.
|
|
197
|
+
- Changing `efConstructionSearch` on an existing index does not trigger a rebuild. Structural parameters (`distance`, `M`, `efConstruction`, `quantization`) do rebuild the index when changed.
|
|
198
|
+
- With `quantization: "int8"`, nearest-neighbor `sort` queries re-rank results against full-precision vectors, restoring exact ordering and exact `$distance` values. Distance-threshold (`lt`/`le`) queries filter on the approximate distance.
|
|
199
|
+
- The correct parameter name is `efConstruction` (seeds the construction budget) and `efConstructionSearch` (controls search budget). The name `efSearchConstruction` is a previous documentation error.
|
|
200
|
+
- When no `ef` is passed and `efConstructionSearch` (or `efConstruction`) is not explicitly set, the search budget auto-scales with index size.
|
|
201
|
+
- `cosine` is the default distance function when `distance` is not specified.
|
package/package.json
CHANGED
package/skills-lock.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"source": "harperfast/skills",
|
|
6
6
|
"sourceType": "github",
|
|
7
7
|
"skillPath": "harper-best-practices/SKILL.md",
|
|
8
|
-
"computedHash": "
|
|
8
|
+
"computedHash": "deb69ad99581b5cc422d76d01fd0554bcd40373f81e10c4ec1f4e0c707d963b7"
|
|
9
9
|
},
|
|
10
10
|
"harper-mcp": {
|
|
11
11
|
"source": "harperfast/skills",
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
name: Deploy to Harper Fabric
|
|
2
|
-
on:
|
|
3
|
-
workflow_dispatch:
|
|
4
|
-
# push:
|
|
5
|
-
# branches:
|
|
6
|
-
# - main
|
|
7
|
-
|
|
8
|
-
concurrency:
|
|
9
|
-
group: main
|
|
10
|
-
cancel-in-progress: false
|
|
11
|
-
|
|
12
|
-
jobs:
|
|
13
|
-
deploy:
|
|
14
|
-
runs-on: ubuntu-latest
|
|
15
|
-
steps:
|
|
16
|
-
- name: Checkout code
|
|
17
|
-
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
|
18
|
-
with:
|
|
19
|
-
fetch-depth: 0
|
|
20
|
-
fetch-tags: true
|
|
21
|
-
- name: Set up Node.js
|
|
22
|
-
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
|
|
23
|
-
with:
|
|
24
|
-
cache: 'npm'
|
|
25
|
-
node-version-file: '.nvmrc'
|
|
26
|
-
- name: Install dependencies
|
|
27
|
-
run: npm ci
|
|
28
|
-
- name: Run unit tests
|
|
29
|
-
run: npm test
|
|
30
|
-
- name: Run lint
|
|
31
|
-
run: npm run lint
|
|
32
|
-
- name: Build & deploy
|
|
33
|
-
run: npm run deploy
|