@pulseindex/sdk 4.0.0 → 5.0.0

package/dist/index.mjs

@@ -42,11 +42,17 @@ var GeoHash = class {
    * How much area outside the circle a covering may carry before a finer
    * precision is worth its cell count.
    *
-   * 2.0 is where the measured choices come out right at every radius: it
-   * rejects the coarse cell at 2 km (6.91x) and 5 km (2.76x) and accepts it at
-   * 15 km (1.44x), which is also where the cell count turns from 47 into 1,120.
+   * `withinRadius` is a pre-filter the caller narrows exactly afterwards, so
+   * excess area is cheaper than predicates.
+   *
+   * 2.0 was the first attempt and it was too strict. It rejected the coarse
+   * cell at 5 km, so a covering that had cost 10 cells cost 167, and the demo
+   * benchmark went from beating PostgreSQL to losing to it by 2.76x on wall
+   * time — the cost is the request, not the search. 3.0 keeps the cheap
+   * covering at 5 km (2.76x) and still rejects the coarse cell at 2 km, where
+   * it wastes 4.73x to 6.91x depending on latitude.
    */
-  static ACCEPTABLE_COVER_RATIO = 2;
+  static ACCEPTABLE_COVER_RATIO = 3;
   static NEIGHBORS = {
     n: ["p0r21436x8zb9dcf5h7kjnmqesgutwvy", "bc01fg45238967deuvhjyznpkmstqrwx"],
     s: ["14365h7k9dcfesgujnmqp0r2twvyx8zb", "238967debc01fg45kmstqrwxuvhjyznp"],
@@ -490,7 +496,6 @@ var FilterOperation = {
   SHOULD: 1,
   MUST_NOT: 2
 };
-var UINT32_MAX = 4294967295;
 var DEFAULT_ENDPOINT = "localhost:50051";
 var DEFAULT_TIMEOUT_MS = 5e3;
 var DEFAULT_POOL_SIZE = 1;
@@ -500,7 +505,8 @@ var DEFAULT_LIMIT = 100;
 function emptyState() {
   return {
     tenantId: "",
-    locationPrefix: "0",
+    exactTotal: false,
+    geo: null,
     limit: DEFAULT_LIMIT,
     offset: 0,
     filters: [],
@@ -536,9 +542,17 @@ var QueryBuilder = class _QueryBuilder {
       state.tenantId = tenantId;
     });
   }
-  location(locationPrefix) {
+  /**
+   * Count every match instead of stopping as soon as the page is full.
+   *
+   * A paged search stops early, so the `totalMatches` it carries is only what
+   * the engine had counted by then — a lower bound, and one that does not look
+   * like one. This makes the count exact in the same request; `totalIsExact`
+   * on the response says which you got.
+   */
+  exactTotal(enabled = true) {
     return this.fork((state) => {
-      state.locationPrefix = String(locationPrefix);
+      state.exactTotal = enabled;
     });
   }
   must(attribute) {
@@ -586,7 +600,11 @@ var QueryBuilder = class _QueryBuilder {
       resolvedPrecision = precision;
     }
     const covering = GeoHash.getCoveringHashes(lat, longitude, radius, resolvedPrecision);
+    const field = typeof latOrOptions === "object" ? latOrOptions.field : void 0;
     return this.fork((state) => {
+      if (field) {
+        state.geo = { field, lat, lon: longitude, radiusKm: radius };
+      }
       const group = state.nextGroup;
       state.nextGroup += 1;
       for (const hash of covering) {
@@ -598,6 +616,15 @@ var QueryBuilder = class _QueryBuilder {
       }
     });
   }
+  /**
+   * Filter on a numeric field's inclusive range.
+   *
+   * `price` is the only number an entity carries. Naming any other field is
+   * refused by the engine rather than answered, because a field nothing carries
+   * can only match nothing, and an empty page looks exactly like a real one.
+   * Model any other number as a category token instead: `must('bedrooms:3')`,
+   * or several in one SHOULD group for a range of values.
+   */
   range(field, min, max) {
     if (!field.trim()) {
       throw new PulseIndexQueryError("Range field must not be empty.");
@@ -645,9 +672,12 @@ var QueryBuilder = class _QueryBuilder {
     return this.sortBy(field, true);
   }
   /**
-   * Order the page by a numeric field. Rows carrying no value for it sort last
-   * in both directions; they still count towards `totalMatches`, they simply
-   * have nothing to be ordered by.
+   * Order the page by a numeric field.
+   *
+   * Bounded exactly as {@link range} is: `price` is the only field an entity
+   * carries, and any other name is refused rather than silently ignored. An
+   * order by a field nothing carries used to leave the page in insertion order
+   * and report it as sorted.
    */
   sortBy(field, descending = false) {
     if (!field.trim()) {
@@ -657,18 +687,62 @@ var QueryBuilder = class _QueryBuilder {
       state.sort = { field, descending };
     });
   }
+  /**
+   * Keep only entities within `radiusKm` of the point, measured exactly.
+   *
+   * This is the circle on its own. {@link withinRadius} with a `field` adds
+   * the geohash cells too, which is what stops the engine opening every part
+   * of the index to find them.
+   */
+  within(field, lat, lon, radiusKm) {
+    if (!field.trim()) {
+      throw new PulseIndexQueryError("A position field name must not be empty.");
+    }
+    if (!Number.isFinite(lat) || !Number.isFinite(lon) || !Number.isFinite(radiusKm)) {
+      throw new PulseIndexQueryError("within(field, lat, lon, radiusKm) needs finite numbers.");
+    }
+    if (radiusKm < 0) {
+      throw new PulseIndexQueryError(`A circle cannot have a radius of ${radiusKm}.`);
+    }
+    return this.fork((state) => {
+      state.geo = { field, lat, lon, radiusKm };
+    });
+  }
+  /**
+   * Order the page by distance from the point, nearest first.
+   *
+   * Without a radius this is "the nearest K of whatever else matched"; combine
+   * it with {@link within} or {@link withinRadius} to bound the search as well.
+   * It used to be impossible: a radius returned everything inside it unordered,
+   * so you hydrated every id from your own store before you could sort them.
+   */
+  nearest(field, lat, lon) {
+    if (!field.trim()) {
+      throw new PulseIndexQueryError("A position field name must not be empty.");
+    }
+    if (!Number.isFinite(lat) || !Number.isFinite(lon)) {
+      throw new PulseIndexQueryError("nearest(field, lat, lon) needs finite numbers.");
+    }
+    return this.fork((state) => {
+      state.geo = { field, lat, lon, radiusKm: state.geo?.radiusKm ?? 0 };
+      state.sort = { field, descending: false, byDistance: true };
+    });
+  }
   toRequest(defaultTenantId = "") {
     const request = {
       tenantId: this.state.tenantId || defaultTenantId,
-      locationPrefix: this.state.locationPrefix,
       limit: this.state.limit,
       offset: this.state.offset,
+      exactTotal: this.state.exactTotal,
       filters: this.state.filters.map((filter) => ({ ...filter })),
       ranges: this.state.ranges.map((range) => ({ ...range }))
     };
     if (this.state.sort) {
       request.sort = { ...this.state.sort };
     }
+    if (this.state.geo) {
+      request.geo = { ...this.state.geo };
+    }
     return request;
   }
   toArray(defaultTenantId = "") {
@@ -687,8 +761,8 @@ var QueryBuilder = class _QueryBuilder {
     if (options.tenantId !== void 0) {
       query = query.tenant(options.tenantId);
     }
-    if (options.locationPrefix !== void 0) {
-      query = query.location(options.locationPrefix);
+    if (options.exactTotal !== void 0) {
+      query = query.exactTotal(options.exactTotal);
     }
     if (options.must !== void 0) {
       query = query.must(options.must);
@@ -737,12 +811,13 @@ var QueryBuilder = class _QueryBuilder {
     const next = new _QueryBuilder(this.executor);
     next.state = {
       tenantId: this.state.tenantId,
-      locationPrefix: this.state.locationPrefix,
       limit: this.state.limit,
       offset: this.state.offset,
+      exactTotal: this.state.exactTotal,
       filters: this.state.filters.map((filter) => ({ ...filter })),
       ranges: this.state.ranges.map((range) => ({ ...range })),
       sort: this.state.sort ? { ...this.state.sort } : null,
+      geo: this.state.geo ? { ...this.state.geo } : null,
       nextGroup: this.state.nextGroup
     };
     mutate(next.state);
@@ -1022,9 +1097,8 @@ var SKIP_ATTRIBUTE_KEYS = /* @__PURE__ */ new Set([
   "entityId",
   "entity_id",
   "attributes",
-  "price",
-  "locationPrefix",
-  "location_prefix",
+  "numbers",
+  "points",
   "tenantId",
   "tenant_id",
   "latitude",
@@ -1056,15 +1130,69 @@ function toUint64String(value, field = "entityId") {
   }
   return trimmed.replace(/^0+(?=\d)/, "");
 }
-function toUint32(value, field) {
-  if (value === void 0 || value === null || value === "") {
-    return 0;
-  }
+function toFieldValue(value, field) {
   const numeric = typeof value === "number" ? value : Number(value);
-  if (!Number.isFinite(numeric) || numeric < 0 || numeric > UINT32_MAX) {
-    throw new PulseIndexQueryError(`${field} must be an integer between 0 and ${UINT32_MAX}.`);
+  if (!Number.isFinite(numeric)) {
+    throw new PulseIndexQueryError(`${field} must be a finite number.`);
+  }
+  if (!Number.isInteger(numeric)) {
+    throw new PulseIndexQueryError(
+      `${field} is ${numeric}, and the engine stores whole numbers. Scale it to an integer and keep the scale on your side \u2014 a price in cents, a rating out of 100.`
+    );
+  }
+  if (!Number.isSafeInteger(numeric)) {
+    throw new PulseIndexQueryError(`${field} is past the range JavaScript can hold exactly.`);
+  }
+  return numeric;
+}
+function collectNumbers(merged) {
+  const source = merged.numbers;
+  if (source === void 0 || source === null) {
+    return {};
+  }
+  if (typeof source !== "object" || Array.isArray(source)) {
+    throw new PulseIndexQueryError("numbers must be an object of field name to number.");
+  }
+  const out = {};
+  for (const [name, value] of Object.entries(source)) {
+    if (!name.trim()) {
+      throw new PulseIndexQueryError("A numeric field name must not be empty.");
+    }
+    if (value === void 0 || value === null || value === "") {
+      continue;
+    }
+    out[name] = toFieldValue(value, name);
+  }
+  return out;
+}
+function collectPoints(merged) {
+  const source = merged.points;
+  if (source === void 0 || source === null) {
+    return {};
+  }
+  if (typeof source !== "object" || Array.isArray(source)) {
+    throw new PulseIndexQueryError("points must be an object of field name to {lat, lon}.");
+  }
+  const out = {};
+  for (const [name, value] of Object.entries(source)) {
+    if (!name.trim()) {
+      throw new PulseIndexQueryError("A position field name must not be empty.");
+    }
+    const point = asRecord(value);
+    const lat = Number(point.lat ?? point.latitude);
+    const lon = Number(point.lon ?? point.lng ?? point.longitude);
+    if (!Number.isFinite(lat) || !Number.isFinite(lon)) {
+      throw new PulseIndexQueryError(`${name} must be {lat, lon} with finite numbers.`);
+    }
+    if (lat < -90 || lat > 90) {
+      throw new PulseIndexQueryError(`${name}.lat is ${lat}, outside -90..90.`);
+    }
+    if (lon < -180 || lon > 180) {
+      throw new PulseIndexQueryError(`${name}.lon is ${lon}, outside -180..180.`);
+    }
+    out[name] = { lat, lon };
   }
-  return Math.floor(numeric);
+  return out;
 }
 function asRecord(value) {
   if (value && typeof value === "object" && !Array.isArray(value)) {
@@ -1170,11 +1298,8 @@ function encodeEntity(entityIdOrInput, attributes = {}, defaults = {}) {
   return {
     entityId: toUint64String(rawId, "entityId"),
     categories: flattenAttributes(merged),
-    price: toUint32(merged.price, "price"),
-    locationPrefix: toUint64String(
-      merged.locationPrefix ?? merged.location_prefix ?? 0,
-      "locationPrefix"
-    ),
+    numbers: collectNumbers(merged),
+    points: collectPoints(merged),
     tenantId
   };
 }
@@ -1208,10 +1333,11 @@ var PulseIndexClient = class _PulseIndexClient {
       matchedEntityIds: (raw.matchedEntityIds ?? []).map((id) => String(id)),
       totalMatches: Number(raw.totalMatches ?? 0),
       executionTimeUs: Number(raw.executionTimeUs ?? 0),
-      // Exact only when nothing made the engine stop early. limit=0 asks for
-      // the count and no ids, and is the only shape that counts every match;
-      // any page can early-exit as soon as it is full.
-      totalIsExact: builder.toArray().limit === 0
+      // The engine says so now. This used to be inferred from `limit === 0`,
+      // which is a rule this SDK had to keep in step with the engine's own by
+      // hand, and which called a page inexact even when every match fit inside
+      // it and nothing was skipped.
+      totalIsExact: Boolean(raw.totalIsExact)
     };
   }
   /**
@@ -1223,22 +1349,14 @@ var PulseIndexClient = class _PulseIndexClient {
    * reported 10,866 for a page of 100. Anything that prints "page 1 of N" from
    * that number is wrong by an order of magnitude and looks fine.
    *
-   * This sends the count query as well, so it costs two round trips and returns
-   * a total you can divide by a page size.
+   * One request. This used to run the whole query twice — once for the page,
+   * once for the count — because the wire had no way to ask for both. It does
+   * now, so this is the same round trip with `exactTotal` set, and the total
+   * you get back can be divided by a page size.
    */
   async searchWithTotal(query) {
-    const page = await this.search(query);
-    if (page.totalIsExact) {
-      return page;
-    }
-    const builder = query instanceof QueryBuilder ? query : QueryBuilder.fromOptions(query);
-    const counted = await this.search(builder.limit(0));
-    return {
-      matchedEntityIds: page.matchedEntityIds,
-      totalMatches: counted.totalMatches,
-      executionTimeUs: page.executionTimeUs + counted.executionTimeUs,
-      totalIsExact: true
-    };
+    const builder = query instanceof QueryBuilder ? query : QueryBuilder.fromOptions(query, this);
+    return this.search(builder.exactTotal());
   }
   async index(entityIdOrInput, attributes = {}) {
     const encoded = encodeEntity(entityIdOrInput, attributes, {
@@ -1249,12 +1367,18 @@ var PulseIndexClient = class _PulseIndexClient {
     );
     return { success: Boolean(raw.success) };
   }
-  async indexEntity(entityId, categories = [], price = 0, locationPrefix = 0, tenantId = "") {
+  /**
+   * Index one record.
+   *
+   * `numbers` are yours to name: `{price_cents: 45000, bedrooms: 3}`. This
+   * used to take a single `price` and a `locationPrefix`, which was a schema
+   * this SDK had no business imposing.
+   */
+  async indexEntity(entityId, categories = [], numbers = {}, tenantId = "") {
     const response = await this.index({
       entityId,
       categories,
-      price,
-      locationPrefix,
+      numbers,
       tenantId: tenantId || this.connection.tenantId
     });
     return response.success;
@@ -1404,8 +1528,8 @@ var PulseIndex = class extends PulseIndexClient {
 function toIndexRequest(encoded) {
   return {
     entityId: encoded.entityId,
-    locationPrefix: encoded.locationPrefix,
-    price: encoded.price,
+    numbers: encoded.numbers,
+    points: encoded.points,
     categories: encoded.categories,
     tenantId: encoded.tenantId
   };

package/package.json

@@ -1,7 +1,11 @@
 {
   "name": "@pulseindex/sdk",
-  "version": "4.0.0",
-  "description": "Official Node.js & TypeScript SDK for PulseIndex — hosted search and filtering for large entity sets",
+  "version": "5.0.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/mohammed-alfarra/pulseindex-js.git"
+  },
+  "description": "Official Node.js & TypeScript SDK for PulseIndex \u2014 hosted search and filtering for large entity sets",
   "license": "MIT",
   "author": "PulseIndex",
   "keywords": [

package/proto/engine.proto

@@ -62,13 +62,34 @@ message IndexEntityRequest {
   // Dense, ascending ids index and query most efficiently.
   uint64 entity_id = 1;
 
-  // Optional geohash-style location prefix. Matches the `location_prefix`
-  // filter on Search, and `location:<prefix>` as an attribute.
-  // Use 0 to omit location indexing.
-  uint64 location_prefix = 2;
+  reserved 2, 3;
+  reserved "location_prefix", "price";
 
-  // Numeric price (or similar scalar), filterable through RangePredicate.
-  uint32 price = 3;
+  // Numeric fields under your own names. Any name, any int64, any number of
+  // them, each filterable through RangePredicate and orderable through
+  // SortSpec. A name means nothing to the engine beyond its hash, so call them
+  // whatever your own records call them:
+  //
+  //   {"price_cents": 45000, "bedrooms": 3, "built_at": 1712000000}
+  //
+  // This replaces a single uint32 named `price` and a uint64 bitfield named
+  // `location_prefix` — one number per record, under a name the engine chose,
+  // with no negatives and nothing past 4,294,967,295.
+  map<string, int64> numbers = 6;
+
+  // Positions, under your own names, the same way `numbers` are.
+  //
+  // The engine packs the pair into one number and keeps it in an ordinary
+  // column, so a position costs what a number costs. Send degrees and let it
+  // pack: a representation split between this SDK and the engine, with nothing
+  // comparing the two, is how the geo bugs in 4.0.0 happened - one side
+  // computes it differently and every answer is a plausible empty page.
+  //
+  //   points: { "where": { lat: 41.0369, lon: 28.9850 } }
+  //
+  // A name used here and in `numbers` on the same record is refused: a field
+  // is either a number or a position.
+  map<string, GeoPoint> points = 7;
 
   // Attribute tokens already namespaced by the client, e.g.:
   //   "feature:pool", "furnishing:full", "amenity:parking"
@@ -81,6 +102,12 @@ message IndexEntityRequest {
 }
 
 // IndexEntityResponse acknowledges a successful upsert.
+// GeoPoint is one position in degrees.
+message GeoPoint {
+  double lat = 1;
+  double lon = 2;
+}
+
 message IndexEntityResponse {
   // True when the entity was accepted into the index.
   // Capacity violations return gRPC RESOURCE_EXHAUSTED instead.
@@ -174,14 +201,20 @@ message FilterPredicate {
 
 // RangePredicate filters a continuous numeric field to an inclusive range.
 message RangePredicate {
-  // Numeric field name. Currently supported: "price".
+  // Numeric field name — whatever you called it in
+  // IndexEntityRequest.numbers.
+  //
+  // A search naming a field no entity in your tenant carries is refused rather
+  // than answered: it can only match nothing, and an empty page looks exactly
+  // like a real one.
   string field = 1;
 
   // Inclusive lower bound of the requested range.
-  uint32 min_val = 2;
+  // int64, so a bound may be negative and may go past 4,294,967,295.
+  int64 min_val = 2;
 
   // Inclusive upper bound of the requested range.
-  uint32 max_val = 3;
+  int64 max_val = 3;
 }
 
 // SortSpec orders a page by a numeric field.
@@ -193,12 +226,47 @@ message RangePredicate {
 // An ordered search cannot stop early, because the best remaining row may be
 // anywhere in the tenant, so it costs more than the same filter unordered.
 // offset + limit is capped at 100,000.
+// GeoPredicate names a position field and a point to measure from.
+//
+// A circle used to reach the engine only as a union of geohash cells this SDK
+// generated, and a union of cells is a superset of the circle: measured at a
+// million entities, a 1 km search returned 2,479 rows where 1,241 were really
+// inside. The cells are still the cheap way to narrow which parts of the index
+// are opened; this is what settles the edge.
+message GeoPredicate {
+  // The position field, named as you named it in IndexEntityRequest.points.
+  string field = 1;
+
+  double lat = 2;
+  double lon = 3;
+
+  // Inclusive radius in kilometres. Zero means no radius filter at all: the
+  // point is then only an origin to measure from, for SortSpec.by_distance.
+  double radius_km = 4;
+}
+
 message SortSpec {
-  // Numeric field name, the same one a RangePredicate would name.
+  // Numeric field name, the same one a RangePredicate would name, and bounded
+  // the same way: a name no entity carries is refused, not silently ignored.
+  // An order by one used to leave the page in insertion order and call it
+  // sorted.
   string field = 1;
 
-  // Largest first when true; smallest first otherwise.
+  // Largest first when true; smallest first otherwise. With by_distance,
+  // false is nearest-first.
   bool descending = 2;
+
+  // Order by distance from GeoPredicate's point rather than by a field value.
+  //
+  // `field` is ignored and the geo predicate must be present. This is what
+  // makes "the nearest fifty" a question you can ask: a radius used to return
+  // everything inside it, unordered, so you had to hydrate every id from your
+  // own store before you could sort them.
+  //
+  // Ordering is to the centimetre, which is the precision a stored position
+  // has. Rows closer together than that tie, and a tie breaks on the entity id
+  // so the same query returns the same page.
+  bool by_distance = 3;
 }
 
 // ---------------------------------------------------------------------------
@@ -207,9 +275,8 @@ message SortSpec {
 
 // SearchQueryRequest executes a filtered search inside one tenant.
 message SearchQueryRequest {
-  // Optional location constraint (0 = no location filter).
-  // When non-zero, only entities indexed with that location_prefix match.
-  uint64 location_prefix = 1;
+  reserved 1;
+  reserved "location_prefix";
 
   // Attribute predicates (MUST / SHOULD / MUST_NOT). Order does not affect the
   // result.
@@ -231,6 +298,19 @@ message SearchQueryRequest {
 
   // Optional ordering. Absent returns matches in entity-id order.
   SortSpec sort = 7;
+
+  // Count every match instead of stopping as soon as the page is full.
+  //
+  // A page query stops early, so the count it carries is only what the chunks
+  // it opened added up to. That is usually what a page wants, and it is much
+  // cheaper. Set this when you need the exact total alongside the page: one
+  // request instead of two, and total_is_exact comes back true.
+  //
+  // limit == 0 already implies it: a count-only request has no page to fill.
+  bool exact_total = 8;
+
+  // Optional circle, and the position field to measure against.
+  GeoPredicate geo = 9;
 }
 
 // SearchQueryResponse returns matched ids and timing metadata.
@@ -239,9 +319,21 @@ message SearchQueryResponse {
   // Caller hydrates full records from the primary data store.
   repeated uint64 matched_entity_ids = 1;
 
-  // Number of matches. Exact when `limit` is 0; may be approximate otherwise.
+  // How many entities matched. Exact when total_is_exact; otherwise a lower
+  // bound - read that field before showing this number to anyone.
   uint32 total_matches = 2;
 
   // Server-side execution time in microseconds (excludes network RTT).
   uint64 execution_time_us = 3;
+
+  // Whether total_matches is the whole answer.
+  //
+  // False means the scan stopped once the page was full, so the count is only
+  // as far as it got. It does not look partial: on 500,000 records that all
+  // matched, a `limit 20` query reported 65,536. Send exact_total to get a
+  // true count in the same request.
+  //
+  // True whenever the scan finished, which includes a page query whose matches
+  // all fit inside it: nothing was skipped, so nothing is missing.
+  bool total_is_exact = 4;
 }