@bhushanpawar/sqldb 1.0.8 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +18 -1
- package/dist/client.js.map +1 -1
- package/dist/query/operations.d.ts +37 -2
- package/dist/query/operations.d.ts.map +1 -1
- package/dist/query/operations.js +55 -2
- package/dist/query/operations.js.map +1 -1
- package/dist/search/geo-search-manager.d.ts +89 -0
- package/dist/search/geo-search-manager.d.ts.map +1 -0
- package/dist/search/geo-search-manager.js +695 -0
- package/dist/search/geo-search-manager.js.map +1 -0
- package/dist/search/geo-utils.d.ts +59 -0
- package/dist/search/geo-utils.d.ts.map +1 -0
- package/dist/search/geo-utils.js +265 -0
- package/dist/search/geo-utils.js.map +1 -0
- package/dist/search/location-normalizer.d.ts +79 -0
- package/dist/search/location-normalizer.d.ts.map +1 -0
- package/dist/search/location-normalizer.js +308 -0
- package/dist/search/location-normalizer.js.map +1 -0
- package/dist/types/geo-search.d.ts +146 -0
- package/dist/types/geo-search.d.ts.map +1 -0
- package/dist/types/geo-search.js +118 -0
- package/dist/types/geo-search.js.map +1 -0
- package/dist/types/query.d.ts +17 -0
- package/dist/types/query.d.ts.map +1 -1
- package/dist/types/search.d.ts +30 -0
- package/dist/types/search.d.ts.map +1 -1
- package/dist/types/search.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,695 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Geographic search manager - combines text search with geospatial capabilities
|
|
4
|
+
*/
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.GeoSearchManager = void 0;
|
|
7
|
+
const geo_search_1 = require("../types/geo-search");
|
|
8
|
+
const geo_utils_1 = require("./geo-utils");
|
|
9
|
+
const location_normalizer_1 = require("./location-normalizer");
|
|
10
|
+
class GeoSearchManager {
|
|
11
|
+
constructor(redis, tableName, config) {
|
|
12
|
+
this.redis = redis;
|
|
13
|
+
this.tableName = tableName;
|
|
14
|
+
this.config = config;
|
|
15
|
+
this.indexKeyPrefix = `geo:${tableName}`;
|
|
16
|
+
// Initialize location normalizer with buckets and mappings
|
|
17
|
+
const buckets = config.buckets || geo_search_1.MAJOR_CITY_BUCKETS;
|
|
18
|
+
const mappings = [
|
|
19
|
+
...(config.locationMappings || []),
|
|
20
|
+
...location_normalizer_1.US_CITY_ALIASES,
|
|
21
|
+
...location_normalizer_1.INTERNATIONAL_CITY_ALIASES,
|
|
22
|
+
];
|
|
23
|
+
this.locationNormalizer = new location_normalizer_1.LocationNormalizer(buckets, mappings);
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Index a document with geographic data
|
|
27
|
+
*/
|
|
28
|
+
async indexDocument(doc) {
|
|
29
|
+
if (!(0, geo_utils_1.isValidCoordinates)(doc.location)) {
|
|
30
|
+
throw new Error(`Invalid coordinates for document ${doc.id}: ${JSON.stringify(doc.location)}`);
|
|
31
|
+
}
|
|
32
|
+
const client = this.redis.getClient();
|
|
33
|
+
if (!client) {
|
|
34
|
+
throw new Error('Redis client not available');
|
|
35
|
+
}
|
|
36
|
+
const pipeline = client.pipeline();
|
|
37
|
+
// 1. Add to main geo index using Redis GEOADD
|
|
38
|
+
const mainGeoKey = `${this.indexKeyPrefix}:main`;
|
|
39
|
+
pipeline.geoadd(mainGeoKey, doc.location.lng, doc.location.lat, String(doc.id));
|
|
40
|
+
// 2. Store document data
|
|
41
|
+
const docKey = `${this.indexKeyPrefix}:doc:${doc.id}`;
|
|
42
|
+
pipeline.set(docKey, JSON.stringify({
|
|
43
|
+
...doc.data,
|
|
44
|
+
_geo_lat: doc.location.lat,
|
|
45
|
+
_geo_lng: doc.location.lng,
|
|
46
|
+
_geo_location_name: doc.locationName,
|
|
47
|
+
_geo_bucket_id: doc.bucketId,
|
|
48
|
+
}));
|
|
49
|
+
// 3. Add to bucket index if applicable
|
|
50
|
+
if (doc.bucketId) {
|
|
51
|
+
const bucketKey = `${this.indexKeyPrefix}:bucket:${doc.bucketId}`;
|
|
52
|
+
pipeline.sadd(bucketKey, String(doc.id));
|
|
53
|
+
}
|
|
54
|
+
else if (this.config.buckets) {
|
|
55
|
+
// Auto-assign bucket
|
|
56
|
+
const bucket = (0, geo_utils_1.findBucket)(doc.location, this.config.buckets);
|
|
57
|
+
if (bucket) {
|
|
58
|
+
const bucketKey = `${this.indexKeyPrefix}:bucket:${bucket.id}`;
|
|
59
|
+
pipeline.sadd(bucketKey, String(doc.id));
|
|
60
|
+
// Update document with bucket ID
|
|
61
|
+
pipeline.hset(docKey, '_geo_bucket_id', bucket.id);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
// 4. Add to location name index if provided
|
|
65
|
+
if (doc.locationName && this.config.autoNormalize) {
|
|
66
|
+
const normalized = this.locationNormalizer.normalize(doc.locationName);
|
|
67
|
+
if (normalized) {
|
|
68
|
+
const locationKey = `${this.indexKeyPrefix}:location:${normalized.canonical}`;
|
|
69
|
+
pipeline.sadd(locationKey, String(doc.id));
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
await pipeline.exec();
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Batch index multiple documents
|
|
76
|
+
*/
|
|
77
|
+
async indexDocuments(docs) {
|
|
78
|
+
const client = this.redis.getClient();
|
|
79
|
+
if (!client) {
|
|
80
|
+
throw new Error('Redis client not available');
|
|
81
|
+
}
|
|
82
|
+
const pipeline = client.pipeline();
|
|
83
|
+
const mainGeoKey = `${this.indexKeyPrefix}:main`;
|
|
84
|
+
for (const doc of docs) {
|
|
85
|
+
if (!(0, geo_utils_1.isValidCoordinates)(doc.location)) {
|
|
86
|
+
console.warn(`Skipping document ${doc.id} with invalid coordinates`);
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
// Add to geo index
|
|
90
|
+
pipeline.geoadd(mainGeoKey, doc.location.lng, doc.location.lat, String(doc.id));
|
|
91
|
+
// Store document
|
|
92
|
+
const docKey = `${this.indexKeyPrefix}:doc:${doc.id}`;
|
|
93
|
+
const docData = {
|
|
94
|
+
...doc.data,
|
|
95
|
+
_geo_lat: doc.location.lat,
|
|
96
|
+
_geo_lng: doc.location.lng,
|
|
97
|
+
};
|
|
98
|
+
if (doc.locationName) {
|
|
99
|
+
docData._geo_location_name = doc.locationName;
|
|
100
|
+
}
|
|
101
|
+
// Auto-assign bucket if enabled
|
|
102
|
+
if (this.config.buckets) {
|
|
103
|
+
const bucket = (0, geo_utils_1.findBucket)(doc.location, this.config.buckets);
|
|
104
|
+
if (bucket) {
|
|
105
|
+
docData._geo_bucket_id = bucket.id;
|
|
106
|
+
const bucketKey = `${this.indexKeyPrefix}:bucket:${bucket.id}`;
|
|
107
|
+
pipeline.sadd(bucketKey, String(doc.id));
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
pipeline.set(docKey, JSON.stringify(docData));
|
|
111
|
+
// Location name indexing
|
|
112
|
+
if (doc.locationName && this.config.autoNormalize) {
|
|
113
|
+
const normalized = this.locationNormalizer.normalize(doc.locationName);
|
|
114
|
+
if (normalized) {
|
|
115
|
+
const locationKey = `${this.indexKeyPrefix}:location:${normalized.canonical}`;
|
|
116
|
+
pipeline.sadd(locationKey, String(doc.id));
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
await pipeline.exec();
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Search for documents within a radius
|
|
124
|
+
* Supports automatic cluster expansion if results are insufficient
|
|
125
|
+
*/
|
|
126
|
+
async searchByRadius(options) {
|
|
127
|
+
const client = this.redis.getClient();
|
|
128
|
+
if (!client) {
|
|
129
|
+
throw new Error('Redis client not available');
|
|
130
|
+
}
|
|
131
|
+
const mainGeoKey = `${this.indexKeyPrefix}:main`;
|
|
132
|
+
// Convert radius to meters for Redis GEORADIUS
|
|
133
|
+
let radiusM;
|
|
134
|
+
switch (options.radius.unit) {
|
|
135
|
+
case 'km':
|
|
136
|
+
radiusM = options.radius.value * 1000;
|
|
137
|
+
break;
|
|
138
|
+
case 'mi':
|
|
139
|
+
radiusM = options.radius.value * 1609.34;
|
|
140
|
+
break;
|
|
141
|
+
default:
|
|
142
|
+
radiusM = options.radius.value;
|
|
143
|
+
}
|
|
144
|
+
// Perform initial geo search
|
|
145
|
+
let results = await client.georadius(mainGeoKey, options.center.lng, options.center.lat, radiusM, 'm', 'WITHDIST', 'ASC', // Sort by distance
|
|
146
|
+
...(options.limit ? ['COUNT', options.limit] : []));
|
|
147
|
+
// Check if cluster expansion is needed
|
|
148
|
+
const minResults = options.minResults ?? 5;
|
|
149
|
+
const hasMaxRange = options.maxRange !== undefined;
|
|
150
|
+
const needsExpansion = hasMaxRange && results.length < minResults;
|
|
151
|
+
if (needsExpansion) {
|
|
152
|
+
// Convert maxRange to meters
|
|
153
|
+
let maxRangeM;
|
|
154
|
+
switch (options.maxRange.unit) {
|
|
155
|
+
case 'km':
|
|
156
|
+
maxRangeM = options.maxRange.value * 1000;
|
|
157
|
+
break;
|
|
158
|
+
case 'mi':
|
|
159
|
+
maxRangeM = options.maxRange.value * 1609.34;
|
|
160
|
+
break;
|
|
161
|
+
default:
|
|
162
|
+
maxRangeM = options.maxRange.value;
|
|
163
|
+
}
|
|
164
|
+
// Only expand if maxRange is larger than current radius
|
|
165
|
+
if (maxRangeM > radiusM) {
|
|
166
|
+
// Expand search to maxRange
|
|
167
|
+
results = await client.georadius(mainGeoKey, options.center.lng, options.center.lat, maxRangeM, 'm', 'WITHDIST', 'ASC', ...(options.limit ? ['COUNT', options.limit] : []));
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
// Process results
|
|
171
|
+
const geoResults = [];
|
|
172
|
+
const originalRadiusM = radiusM;
|
|
173
|
+
for (const result of results) {
|
|
174
|
+
const [id, distanceStr] = result;
|
|
175
|
+
const distance = parseFloat(distanceStr);
|
|
176
|
+
// Fetch document data
|
|
177
|
+
const docKey = `${this.indexKeyPrefix}:doc:${id}`;
|
|
178
|
+
const docDataStr = await client.get(docKey);
|
|
179
|
+
if (!docDataStr)
|
|
180
|
+
continue;
|
|
181
|
+
const docData = JSON.parse(docDataStr);
|
|
182
|
+
// Extract geo metadata
|
|
183
|
+
const { _geo_lat, _geo_lng, _geo_location_name, _geo_bucket_id, ...documentData } = docData;
|
|
184
|
+
// Calculate relevance score based on distance
|
|
185
|
+
// Results within original radius get higher scores
|
|
186
|
+
const maxDistance = needsExpansion ? this.convertToMeters(options.maxRange) : radiusM;
|
|
187
|
+
const baseDistanceScore = 1 - distance / maxDistance;
|
|
188
|
+
// Apply penalty for results outside original radius (if expansion occurred)
|
|
189
|
+
let distanceScore = baseDistanceScore;
|
|
190
|
+
if (needsExpansion && distance > originalRadiusM) {
|
|
191
|
+
// Results beyond original radius get a penalty (0.7x score)
|
|
192
|
+
distanceScore = baseDistanceScore * 0.7;
|
|
193
|
+
}
|
|
194
|
+
// Apply distance boost if configured
|
|
195
|
+
let boost = 1.0;
|
|
196
|
+
if (options.distanceBoost) {
|
|
197
|
+
for (const boostConfig of options.distanceBoost) {
|
|
198
|
+
const boostDistanceM = this.convertToMeters(boostConfig.distance);
|
|
199
|
+
if (distance <= boostDistanceM) {
|
|
200
|
+
boost = Math.max(boost, boostConfig.boost);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
const relevanceScore = distanceScore * boost;
|
|
205
|
+
// Get bucket if available
|
|
206
|
+
let bucket;
|
|
207
|
+
if (_geo_bucket_id && this.config.buckets) {
|
|
208
|
+
bucket = this.config.buckets.find((b) => b.id === _geo_bucket_id);
|
|
209
|
+
}
|
|
210
|
+
geoResults.push({
|
|
211
|
+
document: documentData,
|
|
212
|
+
distance: options.includeDistance !== false ? {
|
|
213
|
+
value: this.convertFromMeters(distance, options.radius.unit),
|
|
214
|
+
unit: options.radius.unit,
|
|
215
|
+
} : undefined,
|
|
216
|
+
bucket,
|
|
217
|
+
relevanceScore,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
return geoResults;
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Search by location name (with normalization)
|
|
224
|
+
*/
|
|
225
|
+
async searchByLocationName(locationName, options) {
|
|
226
|
+
// Normalize location name
|
|
227
|
+
const normalized = this.locationNormalizer.normalize(locationName);
|
|
228
|
+
if (!normalized || !normalized.coordinates) {
|
|
229
|
+
// Try to get from bucket
|
|
230
|
+
const bucket = this.locationNormalizer.getBucket(locationName);
|
|
231
|
+
if (bucket) {
|
|
232
|
+
// Search within bucket
|
|
233
|
+
return this.searchByRadius({
|
|
234
|
+
center: bucket.center,
|
|
235
|
+
radius: bucket.radius,
|
|
236
|
+
...options,
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
throw new Error(`Cannot find coordinates for location: ${locationName}`);
|
|
240
|
+
}
|
|
241
|
+
// Use normalized coordinates for search
|
|
242
|
+
const radius = options?.radius || this.config.defaultRadius || { value: 25, unit: 'km' };
|
|
243
|
+
return this.searchByRadius({
|
|
244
|
+
center: normalized.coordinates,
|
|
245
|
+
radius,
|
|
246
|
+
includeDistance: true,
|
|
247
|
+
sortByDistance: true,
|
|
248
|
+
...options,
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* Search within a specific bucket
|
|
253
|
+
*/
|
|
254
|
+
async searchByBucket(bucketId, limit) {
|
|
255
|
+
const bucket = this.config.buckets?.find((b) => b.id === bucketId);
|
|
256
|
+
if (!bucket) {
|
|
257
|
+
throw new Error(`Bucket not found: ${bucketId}`);
|
|
258
|
+
}
|
|
259
|
+
return this.searchByRadius({
|
|
260
|
+
center: bucket.center,
|
|
261
|
+
radius: bucket.radius,
|
|
262
|
+
bucketId,
|
|
263
|
+
limit,
|
|
264
|
+
includeDistance: true,
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* Build geo-buckets with dynamic clustering
|
|
269
|
+
*/
|
|
270
|
+
async buildGeoBuckets(options) {
|
|
271
|
+
const client = this.redis.getClient();
|
|
272
|
+
if (!client) {
|
|
273
|
+
throw new Error('Redis client not available');
|
|
274
|
+
}
|
|
275
|
+
const mainGeoKey = `${this.indexKeyPrefix}:main`;
|
|
276
|
+
// 1. Fetch all document IDs and their coordinates
|
|
277
|
+
const allMembers = await client.zrange(mainGeoKey, 0, -1);
|
|
278
|
+
if (allMembers.length === 0) {
|
|
279
|
+
return { totalBuckets: 0, buckets: [], avgBucketSize: 0 };
|
|
280
|
+
}
|
|
281
|
+
// 2. Fetch coordinates for all members
|
|
282
|
+
const pipeline = client.pipeline();
|
|
283
|
+
for (const memberId of allMembers) {
|
|
284
|
+
pipeline.geopos(mainGeoKey, memberId);
|
|
285
|
+
}
|
|
286
|
+
const positions = await pipeline.exec();
|
|
287
|
+
// 3. Build document records
|
|
288
|
+
const records = [];
|
|
289
|
+
for (let i = 0; i < allMembers.length; i++) {
|
|
290
|
+
const posResult = positions[i][1];
|
|
291
|
+
// GEOPOS returns an array with [longitude, latitude] as the first element
|
|
292
|
+
const pos = posResult ? posResult[0] : null;
|
|
293
|
+
if (!pos || !Array.isArray(pos) || pos.length !== 2) {
|
|
294
|
+
console.warn(`No valid position data for member ${allMembers[i]}, got:`, posResult);
|
|
295
|
+
continue;
|
|
296
|
+
}
|
|
297
|
+
const [lngStr, latStr] = pos;
|
|
298
|
+
const lng = parseFloat(lngStr);
|
|
299
|
+
const lat = parseFloat(latStr);
|
|
300
|
+
// Skip records with invalid coordinates
|
|
301
|
+
if (isNaN(lat) || isNaN(lng)) {
|
|
302
|
+
console.warn(`Skipping member ${allMembers[i]} with invalid coordinates: lat=${latStr}→${lat}, lng=${lngStr}→${lng}`);
|
|
303
|
+
continue;
|
|
304
|
+
}
|
|
305
|
+
// Fetch location name if available
|
|
306
|
+
const docKey = `${this.indexKeyPrefix}:doc:${allMembers[i]}`;
|
|
307
|
+
const docDataStr = await client.get(docKey);
|
|
308
|
+
let locationName;
|
|
309
|
+
if (docDataStr) {
|
|
310
|
+
const docData = JSON.parse(docDataStr);
|
|
311
|
+
locationName = docData._geo_location_name;
|
|
312
|
+
}
|
|
313
|
+
records.push({ id: allMembers[i], lat, lng, locationName });
|
|
314
|
+
}
|
|
315
|
+
// 4. Create grid-based clusters
|
|
316
|
+
const gridSizeDeg = options.gridSizeKm / 111; // Approximate km to degrees
|
|
317
|
+
const gridCells = new Map();
|
|
318
|
+
for (const record of records) {
|
|
319
|
+
const cellLat = Math.floor(record.lat / gridSizeDeg) * gridSizeDeg;
|
|
320
|
+
const cellLng = Math.floor(record.lng / gridSizeDeg) * gridSizeDeg;
|
|
321
|
+
const cellKey = `${cellLat.toFixed(4)},${cellLng.toFixed(4)}`;
|
|
322
|
+
if (!gridCells.has(cellKey)) {
|
|
323
|
+
gridCells.set(cellKey, []);
|
|
324
|
+
}
|
|
325
|
+
gridCells.get(cellKey).push(record);
|
|
326
|
+
}
|
|
327
|
+
// 5. Process cells into buckets
|
|
328
|
+
const buckets = [];
|
|
329
|
+
let bucketIdCounter = 1;
|
|
330
|
+
for (const [cellKey, cellRecords] of gridCells.entries()) {
|
|
331
|
+
if (cellRecords.length < options.minBucketSize) {
|
|
332
|
+
// Too few items, skip or merge later
|
|
333
|
+
continue;
|
|
334
|
+
}
|
|
335
|
+
if (cellRecords.length > options.targetBucketSize * 3) {
|
|
336
|
+
// Cell too large, subdivide using k-means
|
|
337
|
+
const numClusters = Math.ceil(cellRecords.length / options.targetBucketSize);
|
|
338
|
+
const subClusters = this.kMeansClustering(cellRecords, numClusters);
|
|
339
|
+
for (const cluster of subClusters) {
|
|
340
|
+
if (cluster.length >= options.minBucketSize) {
|
|
341
|
+
buckets.push(this.createBucket(cluster, bucketIdCounter++));
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
else {
|
|
346
|
+
// Cell size is reasonable
|
|
347
|
+
buckets.push(this.createBucket(cellRecords, bucketIdCounter++));
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
// 6. Store buckets in Redis
|
|
351
|
+
// IMPORTANT: ioredis SCAN needs manual prefix, but DEL auto-prefixes
|
|
352
|
+
const keyPrefix = client.options?.keyPrefix || '';
|
|
353
|
+
// Clear existing buckets FIRST (not in pipeline) - Use SCAN instead of KEYS
|
|
354
|
+
const oldBucketKeys = [];
|
|
355
|
+
const oldBucketDataKeys = [];
|
|
356
|
+
// Scan for old bucket keys - SCAN pattern needs manual prefix
|
|
357
|
+
let bucketCursor = '0';
|
|
358
|
+
do {
|
|
359
|
+
const [nextCursor, matchedKeys] = await client.scan(bucketCursor, 'MATCH', `${keyPrefix}${this.indexKeyPrefix}:bucket:*`, 'COUNT', 100);
|
|
360
|
+
bucketCursor = nextCursor;
|
|
361
|
+
oldBucketKeys.push(...matchedKeys);
|
|
362
|
+
} while (bucketCursor !== '0');
|
|
363
|
+
// Scan for old bucket-data keys - SCAN pattern needs manual prefix
|
|
364
|
+
let dataCursor = '0';
|
|
365
|
+
do {
|
|
366
|
+
const [nextCursor, matchedKeys] = await client.scan(dataCursor, 'MATCH', `${keyPrefix}${this.indexKeyPrefix}:bucket-data:*`, 'COUNT', 100);
|
|
367
|
+
dataCursor = nextCursor;
|
|
368
|
+
oldBucketDataKeys.push(...matchedKeys);
|
|
369
|
+
} while (dataCursor !== '0');
|
|
370
|
+
console.log(`🧹 Clearing ${oldBucketKeys.length} old bucket keys and ${oldBucketDataKeys.length} old bucket-data keys`);
|
|
371
|
+
// DEL auto-adds prefix, so strip it from SCAN results
|
|
372
|
+
if (oldBucketKeys.length > 0) {
|
|
373
|
+
const keysWithoutPrefix = oldBucketKeys.map(k => k.startsWith(keyPrefix) ? k.substring(keyPrefix.length) : k);
|
|
374
|
+
await client.del(...keysWithoutPrefix);
|
|
375
|
+
}
|
|
376
|
+
if (oldBucketDataKeys.length > 0) {
|
|
377
|
+
const keysWithoutPrefix = oldBucketDataKeys.map(k => k.startsWith(keyPrefix) ? k.substring(keyPrefix.length) : k);
|
|
378
|
+
await client.del(...keysWithoutPrefix);
|
|
379
|
+
}
|
|
380
|
+
// Now create pipeline for storing new buckets
|
|
381
|
+
const storePipeline = client.pipeline();
|
|
382
|
+
// Store new buckets
|
|
383
|
+
console.log(`💾 Storing ${buckets.length} new geo-buckets with prefix: ${this.indexKeyPrefix}`);
|
|
384
|
+
console.log(` Redis client status: ${client.status}`);
|
|
385
|
+
console.log(` Sample key to store: ${this.indexKeyPrefix}:bucket-data:bucket_1`);
|
|
386
|
+
for (const bucket of buckets) {
|
|
387
|
+
const bucketKey = `${this.indexKeyPrefix}:bucket:${bucket.id}`;
|
|
388
|
+
const bucketDataKey = `${this.indexKeyPrefix}:bucket-data:${bucket.id}`;
|
|
389
|
+
// Store member IDs (only if members exist)
|
|
390
|
+
if (bucket.members && bucket.members.length > 0) {
|
|
391
|
+
storePipeline.sadd(bucketKey, ...bucket.members);
|
|
392
|
+
}
|
|
393
|
+
// Store bucket metadata
|
|
394
|
+
storePipeline.set(bucketDataKey, JSON.stringify({
|
|
395
|
+
id: bucket.id,
|
|
396
|
+
center: bucket.center,
|
|
397
|
+
radius: bucket.radius,
|
|
398
|
+
count: bucket.members?.length || 0,
|
|
399
|
+
locationName: bucket.locationName,
|
|
400
|
+
bounds: bucket.bounds,
|
|
401
|
+
}));
|
|
402
|
+
}
|
|
403
|
+
const pipelineResults = await storePipeline.exec();
|
|
404
|
+
console.log(`✅ Stored buckets to Redis. Pipeline executed ${pipelineResults?.length || 0} commands`);
|
|
405
|
+
// Check for errors in pipeline results
|
|
406
|
+
if (pipelineResults) {
|
|
407
|
+
const errors = pipelineResults.filter(([err]) => err !== null);
|
|
408
|
+
if (errors.length > 0) {
|
|
409
|
+
console.error(`❌ Pipeline execution had ${errors.length} errors:`, errors.slice(0, 3));
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
// Verify storage using SCAN instead of KEYS - SCAN pattern needs manual prefix
|
|
413
|
+
const verifyKeys = [];
|
|
414
|
+
let verifyCursor = '0';
|
|
415
|
+
do {
|
|
416
|
+
const [nextCursor, matchedKeys] = await client.scan(verifyCursor, 'MATCH', `${keyPrefix}${this.indexKeyPrefix}:bucket-data:*`, 'COUNT', 100);
|
|
417
|
+
verifyCursor = nextCursor;
|
|
418
|
+
verifyKeys.push(...matchedKeys);
|
|
419
|
+
} while (verifyCursor !== '0');
|
|
420
|
+
console.log(`✔️ Verification: ${verifyKeys.length} bucket-data keys now exist in Redis`);
|
|
421
|
+
if (verifyKeys.length === 0 && buckets.length > 0) {
|
|
422
|
+
console.error(`⚠️ STORAGE FAILED! Expected ${buckets.length} keys but found 0`);
|
|
423
|
+
console.error(` Index key prefix: ${this.indexKeyPrefix}`);
|
|
424
|
+
console.error(` Redis keyPrefix: ${keyPrefix}`);
|
|
425
|
+
console.error(` Sample bucket key would be: ${keyPrefix}${this.indexKeyPrefix}:bucket-data:bucket_1`);
|
|
426
|
+
// Try to manually check one key - GET auto-adds prefix
|
|
427
|
+
const testKey = `${this.indexKeyPrefix}:bucket-data:bucket_1`;
|
|
428
|
+
const testValue = await client.get(testKey);
|
|
429
|
+
console.error(` Manual check of ${testKey}: ${testValue ? 'EXISTS' : 'NOT FOUND'}`);
|
|
430
|
+
}
|
|
431
|
+
// Calculate stats
|
|
432
|
+
const totalItems = buckets.reduce((sum, b) => sum + (b.members?.length || 0), 0);
|
|
433
|
+
const avgBucketSize = buckets.length > 0 ? totalItems / buckets.length : 0;
|
|
434
|
+
return {
|
|
435
|
+
totalBuckets: buckets.length,
|
|
436
|
+
buckets,
|
|
437
|
+
avgBucketSize: parseFloat(avgBucketSize.toFixed(1)),
|
|
438
|
+
};
|
|
439
|
+
}
|
|
440
|
+
/**
|
|
441
|
+
* Get all geo-buckets for this table
|
|
442
|
+
*/
|
|
443
|
+
async getGeoBuckets() {
|
|
444
|
+
const client = this.redis.getClient();
|
|
445
|
+
if (!client) {
|
|
446
|
+
throw new Error('Redis client not available');
|
|
447
|
+
}
|
|
448
|
+
// IMPORTANT: ioredis SCAN command does NOT auto-prefix patterns
|
|
449
|
+
// We need to manually add the keyPrefix for SCAN patterns
|
|
450
|
+
const keyPrefix = client.options?.keyPrefix || '';
|
|
451
|
+
const searchPattern = `${keyPrefix}${this.indexKeyPrefix}:bucket-data:*`;
|
|
452
|
+
console.log(`🔍 Searching for geo-buckets with pattern: ${searchPattern}`);
|
|
453
|
+
// Use SCAN instead of KEYS - SCAN patterns need manual prefix
|
|
454
|
+
const bucketDataKeys = [];
|
|
455
|
+
let cursor = '0';
|
|
456
|
+
do {
|
|
457
|
+
const [nextCursor, matchedKeys] = await client.scan(cursor, 'MATCH', searchPattern, 'COUNT', 100);
|
|
458
|
+
cursor = nextCursor;
|
|
459
|
+
bucketDataKeys.push(...matchedKeys);
|
|
460
|
+
} while (cursor !== '0');
|
|
461
|
+
console.log(`📋 Found ${bucketDataKeys.length} bucket data keys:`, bucketDataKeys.slice(0, 5));
|
|
462
|
+
if (bucketDataKeys.length === 0) {
|
|
463
|
+
// Check if any geo keys exist at all using SCAN
|
|
464
|
+
const allGeoKeys = [];
|
|
465
|
+
let geoCursor = '0';
|
|
466
|
+
do {
|
|
467
|
+
const [nextCursor, matchedKeys] = await client.scan(geoCursor, 'MATCH', `${keyPrefix}${this.indexKeyPrefix}:*`, 'COUNT', 100);
|
|
468
|
+
geoCursor = nextCursor;
|
|
469
|
+
allGeoKeys.push(...matchedKeys);
|
|
470
|
+
} while (geoCursor !== '0' && allGeoKeys.length < 20);
|
|
471
|
+
console.log(`ℹ️ Total geo keys for ${this.tableName}:`, allGeoKeys.length);
|
|
472
|
+
if (allGeoKeys.length > 0) {
|
|
473
|
+
console.log(` Sample keys:`, allGeoKeys.slice(0, 10));
|
|
474
|
+
}
|
|
475
|
+
return [];
|
|
476
|
+
}
|
|
477
|
+
// For GET commands, ioredis auto-adds prefix, so we need to strip it from SCAN results
|
|
478
|
+
const pipeline = client.pipeline();
|
|
479
|
+
for (const key of bucketDataKeys) {
|
|
480
|
+
// Remove the prefix from the key since GET will add it back
|
|
481
|
+
const keyWithoutPrefix = key.startsWith(keyPrefix) ? key.substring(keyPrefix.length) : key;
|
|
482
|
+
pipeline.get(keyWithoutPrefix);
|
|
483
|
+
}
|
|
484
|
+
const results = await pipeline.exec();
|
|
485
|
+
const buckets = [];
|
|
486
|
+
for (const result of results) {
|
|
487
|
+
const bucketDataStr = result[1];
|
|
488
|
+
if (bucketDataStr) {
|
|
489
|
+
buckets.push(JSON.parse(bucketDataStr));
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
console.log(`✅ Retrieved ${buckets.length} geo-buckets from Redis`);
|
|
493
|
+
return buckets;
|
|
494
|
+
}
|
|
495
|
+
/**
|
|
496
|
+
* Simple k-means clustering
|
|
497
|
+
*/
|
|
498
|
+
kMeansClustering(records, k) {
|
|
499
|
+
if (records.length <= k) {
|
|
500
|
+
return records.map(r => [r]);
|
|
501
|
+
}
|
|
502
|
+
// Initialize centroids randomly
|
|
503
|
+
const centroids = [];
|
|
504
|
+
const usedIndices = new Set();
|
|
505
|
+
for (let i = 0; i < k; i++) {
|
|
506
|
+
let idx;
|
|
507
|
+
do {
|
|
508
|
+
idx = Math.floor(Math.random() * records.length);
|
|
509
|
+
} while (usedIndices.has(idx));
|
|
510
|
+
usedIndices.add(idx);
|
|
511
|
+
centroids.push({ lat: records[idx].lat, lng: records[idx].lng });
|
|
512
|
+
}
|
|
513
|
+
// Iterate until convergence (max 20 iterations)
|
|
514
|
+
for (let iter = 0; iter < 20; iter++) {
|
|
515
|
+
// Assign records to nearest centroid
|
|
516
|
+
const clusters = Array.from({ length: k }, () => []);
|
|
517
|
+
for (const record of records) {
|
|
518
|
+
let minDist = Infinity;
|
|
519
|
+
let nearestCluster = 0;
|
|
520
|
+
for (let i = 0; i < k; i++) {
|
|
521
|
+
const dist = (0, geo_utils_1.calculateDistance)({ lat: record.lat, lng: record.lng }, centroids[i]);
|
|
522
|
+
if (dist < minDist) {
|
|
523
|
+
minDist = dist;
|
|
524
|
+
nearestCluster = i;
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
clusters[nearestCluster].push(record);
|
|
528
|
+
}
|
|
529
|
+
// Recalculate centroids
|
|
530
|
+
let changed = false;
|
|
531
|
+
for (let i = 0; i < k; i++) {
|
|
532
|
+
if (clusters[i].length === 0)
|
|
533
|
+
continue;
|
|
534
|
+
const newLat = clusters[i].reduce((sum, r) => sum + r.lat, 0) / clusters[i].length;
|
|
535
|
+
const newLng = clusters[i].reduce((sum, r) => sum + r.lng, 0) / clusters[i].length;
|
|
536
|
+
if (Math.abs(newLat - centroids[i].lat) > 0.0001 || Math.abs(newLng - centroids[i].lng) > 0.0001) {
|
|
537
|
+
changed = true;
|
|
538
|
+
}
|
|
539
|
+
centroids[i] = { lat: newLat, lng: newLng };
|
|
540
|
+
}
|
|
541
|
+
if (!changed)
|
|
542
|
+
break;
|
|
543
|
+
}
|
|
544
|
+
// Return clusters with at least one member
|
|
545
|
+
const clusters = Array.from({ length: k }, () => []);
|
|
546
|
+
for (const record of records) {
|
|
547
|
+
let minDist = Infinity;
|
|
548
|
+
let nearestCluster = 0;
|
|
549
|
+
for (let i = 0; i < k; i++) {
|
|
550
|
+
const dist = (0, geo_utils_1.calculateDistance)({ lat: record.lat, lng: record.lng }, centroids[i]);
|
|
551
|
+
if (dist < minDist) {
|
|
552
|
+
minDist = dist;
|
|
553
|
+
nearestCluster = i;
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
clusters[nearestCluster].push(record);
|
|
557
|
+
}
|
|
558
|
+
return clusters.filter(c => c.length > 0);
|
|
559
|
+
}
|
|
560
|
+
/**
|
|
561
|
+
* Create a bucket from a cluster of records
|
|
562
|
+
*/
|
|
563
|
+
createBucket(records, bucketId) {
|
|
564
|
+
// Calculate centroid
|
|
565
|
+
const centerLat = records.reduce((sum, r) => sum + r.lat, 0) / records.length;
|
|
566
|
+
const centerLng = records.reduce((sum, r) => sum + r.lng, 0) / records.length;
|
|
567
|
+
const center = { lat: centerLat, lng: centerLng };
|
|
568
|
+
// Calculate max distance from center (for radius)
|
|
569
|
+
let maxDist = 0;
|
|
570
|
+
for (const record of records) {
|
|
571
|
+
const dist = (0, geo_utils_1.calculateDistance)(center, { lat: record.lat, lng: record.lng });
|
|
572
|
+
maxDist = Math.max(maxDist, dist);
|
|
573
|
+
}
|
|
574
|
+
// Add 10% buffer to radius
|
|
575
|
+
const radiusKm = maxDist * 1.1;
|
|
576
|
+
// Calculate bounding box
|
|
577
|
+
const lats = records.map(r => r.lat);
|
|
578
|
+
const lngs = records.map(r => r.lng);
|
|
579
|
+
const bounds = {
|
|
580
|
+
northEast: {
|
|
581
|
+
lat: Math.max(...lats),
|
|
582
|
+
lng: Math.max(...lngs),
|
|
583
|
+
},
|
|
584
|
+
southWest: {
|
|
585
|
+
lat: Math.min(...lats),
|
|
586
|
+
lng: Math.min(...lngs),
|
|
587
|
+
},
|
|
588
|
+
};
|
|
589
|
+
// Determine primary location name (most common)
|
|
590
|
+
const locationCounts = {};
|
|
591
|
+
for (const record of records) {
|
|
592
|
+
if (record.locationName) {
|
|
593
|
+
const loc = record.locationName.toLowerCase();
|
|
594
|
+
locationCounts[loc] = (locationCounts[loc] || 0) + 1;
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
const locationName = Object.keys(locationCounts).length > 0
|
|
598
|
+
? Object.entries(locationCounts)
|
|
599
|
+
.sort(([, a], [, b]) => b - a)[0][0]
|
|
600
|
+
: undefined;
|
|
601
|
+
return {
|
|
602
|
+
id: `bucket_${bucketId}`,
|
|
603
|
+
center,
|
|
604
|
+
radius: { value: radiusKm, unit: 'km' },
|
|
605
|
+
members: records.map(r => r.id),
|
|
606
|
+
locationName,
|
|
607
|
+
bounds,
|
|
608
|
+
};
|
|
609
|
+
}
|
|
610
|
+
/**
|
|
611
|
+
* Get index statistics
|
|
612
|
+
*/
|
|
613
|
+
async getStats() {
|
|
614
|
+
const client = this.redis.getClient();
|
|
615
|
+
if (!client) {
|
|
616
|
+
throw new Error('Redis client not available');
|
|
617
|
+
}
|
|
618
|
+
const mainGeoKey = `${this.indexKeyPrefix}:main`;
|
|
619
|
+
// Get total documents
|
|
620
|
+
const totalDocuments = await client.zcard(mainGeoKey);
|
|
621
|
+
// Get bucket counts
|
|
622
|
+
const bucketCounts = {};
|
|
623
|
+
if (this.config.buckets) {
|
|
624
|
+
for (const bucket of this.config.buckets) {
|
|
625
|
+
const bucketKey = `${this.indexKeyPrefix}:bucket:${bucket.id}`;
|
|
626
|
+
const count = await client.scard(bucketKey);
|
|
627
|
+
bucketCounts[bucket.id] = count;
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
// Calculate geographic bounds (would require fetching all docs - expensive)
|
|
631
|
+
// For now, return null and calculate on demand if needed
|
|
632
|
+
const bounds = {
|
|
633
|
+
northEast: { lat: 90, lng: 180 },
|
|
634
|
+
southWest: { lat: -90, lng: -180 },
|
|
635
|
+
};
|
|
636
|
+
// Get index size estimate
|
|
637
|
+
const keys = await client.keys(`${this.indexKeyPrefix}:*`);
|
|
638
|
+
let indexSize = 0;
|
|
639
|
+
for (const key of keys.slice(0, 100)) { // Sample first 100 keys
|
|
640
|
+
const memory = await client.memory('USAGE', key);
|
|
641
|
+
if (memory)
|
|
642
|
+
indexSize += memory;
|
|
643
|
+
}
|
|
644
|
+
indexSize = Math.round((indexSize / Math.min(keys.length, 100)) * keys.length);
|
|
645
|
+
return {
|
|
646
|
+
totalDocuments,
|
|
647
|
+
bucketCounts,
|
|
648
|
+
bounds,
|
|
649
|
+
normalizedLocations: this.locationNormalizer.getStats().totalCanonical,
|
|
650
|
+
indexSize,
|
|
651
|
+
lastUpdated: new Date(),
|
|
652
|
+
};
|
|
653
|
+
}
|
|
654
|
+
/**
|
|
655
|
+
* Clear all geo indexes for this table
|
|
656
|
+
*/
|
|
657
|
+
async clearIndex() {
|
|
658
|
+
const client = this.redis.getClient();
|
|
659
|
+
if (!client) {
|
|
660
|
+
throw new Error('Redis client not available');
|
|
661
|
+
}
|
|
662
|
+
const keys = await client.keys(`${this.indexKeyPrefix}:*`);
|
|
663
|
+
if (keys.length > 0) {
|
|
664
|
+
await client.del(...keys);
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
/**
|
|
668
|
+
* Helper: Convert distance to meters
|
|
669
|
+
*/
|
|
670
|
+
convertToMeters(distance) {
|
|
671
|
+
switch (distance.unit) {
|
|
672
|
+
case 'km':
|
|
673
|
+
return distance.value * 1000;
|
|
674
|
+
case 'mi':
|
|
675
|
+
return distance.value * 1609.34;
|
|
676
|
+
default:
|
|
677
|
+
return distance.value;
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
/**
|
|
681
|
+
* Helper: Convert meters to target unit
|
|
682
|
+
*/
|
|
683
|
+
convertFromMeters(meters, unit) {
|
|
684
|
+
switch (unit) {
|
|
685
|
+
case 'km':
|
|
686
|
+
return meters / 1000;
|
|
687
|
+
case 'mi':
|
|
688
|
+
return meters / 1609.34;
|
|
689
|
+
default:
|
|
690
|
+
return meters;
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
exports.GeoSearchManager = GeoSearchManager;
|
|
695
|
+
//# sourceMappingURL=geo-search-manager.js.map
|