openchemlib-search-wasm 1.2.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +290 -3
  2. package/lib/functionKeys/dictionary.json +72071 -0
  3. package/lib/functionKeys/index.d.ts +105 -0
  4. package/lib/functionKeys/index.d.ts.map +1 -0
  5. package/lib/functionKeys/index.js +98 -0
  6. package/lib/functionKeys/index.js.map +1 -0
  7. package/lib/functions.d.ts +38 -0
  8. package/lib/functions.d.ts.map +1 -0
  9. package/lib/functions.js +54 -0
  10. package/lib/functions.js.map +1 -0
  11. package/lib/hex.d.ts +31 -0
  12. package/lib/hex.d.ts.map +1 -0
  13. package/lib/hex.js +41 -0
  14. package/lib/hex.js.map +1 -0
  15. package/lib/index.d.ts +140 -2
  16. package/lib/index.d.ts.map +1 -1
  17. package/lib/index.js +184 -7
  18. package/lib/index.js.map +1 -1
  19. package/lib/records.d.ts +45 -0
  20. package/lib/records.d.ts.map +1 -0
  21. package/lib/records.js +96 -0
  22. package/lib/records.js.map +1 -0
  23. package/lib/scan.d.ts +99 -7
  24. package/lib/scan.d.ts.map +1 -1
  25. package/lib/scan.js +158 -11
  26. package/lib/scan.js.map +1 -1
  27. package/lib/strongHash.d.ts +9 -0
  28. package/lib/strongHash.d.ts.map +1 -0
  29. package/lib/strongHash.js +56 -0
  30. package/lib/strongHash.js.map +1 -0
  31. package/lib/substructureSearch.d.ts +8 -5
  32. package/lib/substructureSearch.d.ts.map +1 -1
  33. package/lib/substructureSearch.js +28 -5
  34. package/lib/substructureSearch.js.map +1 -1
  35. package/lib/types.d.ts +120 -3
  36. package/lib/types.d.ts.map +1 -1
  37. package/lib/types.js +17 -1
  38. package/lib/types.js.map +1 -1
  39. package/lib/wasm/load.d.ts.map +1 -1
  40. package/lib/wasm/load.js +9 -1
  41. package/lib/wasm/load.js.map +1 -1
  42. package/lib/wasm/log.d.ts +36 -0
  43. package/lib/wasm/log.d.ts.map +1 -0
  44. package/lib/wasm/log.js +52 -0
  45. package/lib/wasm/log.js.map +1 -0
  46. package/package.json +6 -3
  47. package/src/functionKeys/index.ts +181 -0
  48. package/src/functions.ts +56 -0
  49. package/src/hex.ts +43 -0
  50. package/src/index.ts +269 -5
  51. package/src/records.ts +142 -0
  52. package/src/scan.ts +259 -16
  53. package/src/strongHash.ts +57 -0
  54. package/src/substructureSearch.ts +29 -6
  55. package/src/types.ts +171 -2
  56. package/src/wasm/load.ts +10 -1
  57. package/src/wasm/log.ts +66 -0
  58. package/wasm/data.js +1 -1
  59. package/wasm/runtime.d.ts +6 -0
package/README.md CHANGED
@@ -21,7 +21,8 @@ maintainer job**: the `openchemlib` git submodule and the JDK it needs live in
21
21
  ## Use
22
22
 
23
23
  ```ts
24
- substructureSearch<Entry>(idCodeQuery: string, entries: Entry[],
24
+ // idCodeQuery: one idcode, or several — a molecule then matches when it holds any of them
25
+ substructureSearch<Entry>(idCodeQuery: string | string[], entries: Entry[],
25
26
  options?: SubstructureSearchOptions): SubstructureSearchResult<Entry>;
26
27
  similaritySearch<Entry>(idCodeQuery: string, entries: Entry[],
27
28
  options?: SimilaritySearchOptions): SimilaritySearchResult<Entry>;
@@ -30,13 +31,54 @@ similaritySearch<Entry>(idCodeQuery: string, entries: Entry[],
30
31
  getIndex(idCode: string): Int32Array;
31
32
  getIndexes(idCodes: string[]): Int32Array[];
32
33
 
33
- // the 64-bit hash a molecule keeps across its stereoisomers
34
+ // the 512-bit screen next to it, from a dictionary of 512 query fragments
35
+ getScreen(idCode: string): Int32Array;
36
+ getScreens(idCodes: string[]): Int32Array[];
37
+
38
+ // what every molecule a query matches has in its FragFp and in its screen
39
+ getQueryIndex(idCodeQuery: string): Int32Array;
40
+ getQueryScreen(idCodeQuery: string): Int32Array;
41
+
42
+ // the 512 key fragments behind the bits of each, in bit order
43
+ getFragFpKeys(): string[];
44
+ getScreenKeys(): string[];
45
+
46
+ // FUNCTIONS ARE NOT YET WELL DEFINED AND MAY / WILL CHANGE ANYTIME WITHOUT BREAKING CHANGE !
47
+ // 512 named function bits — thioether, aromatic ring, uracil, Boc — and their fragments
48
+ getFunction(idCode: string): Int32Array;
49
+ getFunctions(idCodes: string[]): Int32Array[];
50
+ getFunctionKeyFragments(): string[][];
51
+
52
+
34
53
  getNoStereoHash(idCode: string, options?: HashOptions): bigint;
35
54
  getNoStereoHashes(idCodes: string[], options?: HashOptions): BigInt64Array;
36
55
 
37
56
  // the same, across its tautomers as well
38
57
  getNoStereoTautomerHash(idCode: string, options?: HashOptions): bigint;
39
58
  getNoStereoTautomerHashes(idCodes: string[], options?: HashOptions): BigInt64Array;
59
+
60
+ // the same two canonical forms as idcodes, for a column that stores the form itself
61
+ getNoStereoIdCode(idCode: string, options?: HashOptions): string;
62
+ getNoStereoIdCodes(idCodes: string[], options?: HashOptions): string[];
63
+ getNoStereoTautomerIdCode(idCode: string, options?: HashOptions): string;
64
+ getNoStereoTautomerIdCodes(idCodes: string[], options?: HashOptions): string[];
65
+
66
+ // everything a molecule table stores, from one parse per molecule
67
+ getMoleculeRecords(idCodes: string[], options?: MoleculeRecordsOptions): MoleculeRecords;
68
+ // only the atom count and the relative weight of each molecule
69
+ getMoleculeSizes(idCodes: string[]): MoleculeSizes;
70
+ // the fewest atoms a molecule needs to contain the query
71
+ getQueryAtomFloor(idCodeQuery: string): number;
72
+
73
+ // a hash as the 16 hex digits that cross an API, and back
74
+ hashToHex(hash: bigint): string;
75
+ hexToHash(hex: string): bigint;
76
+
77
+ // the hash of a canonical idcode you already hold, without canonizing again
78
+ strongHash(idCode: string): bigint;
79
+
80
+ // where OpenChemLib's own diagnostics go; nothing is printed by default
81
+ setLogHandler(handler: LogHandler | null): void;
40
82
  ```
41
83
 
42
84
  **Give it your objects and it gives them back.** An entry is an idcode or anything carrying one — a
@@ -144,8 +186,94 @@ const index = getIndex(idCode);
144
186
  const columns = new BigInt64Array(index.buffer, index.byteOffset, 8);
145
187
  ```
146
188
 
189
+ **Pass `byteOffset` and the length, as above.** `getIndexes` returns its views over one shared
190
+ buffer — offsets 0, 64, 128 and so on — so `new BigInt64Array(index.buffer)` is 8 columns only for
191
+ the first molecule and silently reads the first molecule's words for every other one. It is wrong in
192
+ a way nothing reports: the row is written, the index is built, and the screen then misses real hits.
193
+ `getIndex` on its own always sits at offset 0, which is exactly what makes the mistake survive
194
+ testing on one molecule.
195
+
147
196
  This is the expensive half of importing a library, and where the biggest speedup is.
148
197
 
198
+ ## Screens
199
+
200
+ FragFp alone lets many molecules through a prescreen that a substructure search then rejects.
201
+ `getScreen` adds 512 more bits: bit i is set when key i of a frozen dictionary of 512 OpenChemLib
202
+ query fragments is a substructure of the molecule, matched exactly as FragFp matches its own keys. A
203
+ molecule is a candidate for a query only if it holds every bit of the query's FragFp and of its
204
+ screen:
205
+
206
+ ```js
207
+ const isSubset = (query, molecule) =>
208
+ query.every((word, i) => (word & molecule[i]) === word);
209
+
210
+ const isCandidate =
211
+ isSubset(getQueryIndex(query), getIndex(idCode)) &&
212
+ isSubset(getQueryScreen(query), getScreen(idCode));
213
+ ```
214
+
215
+ On 1 M PubChem molecules, requiring both lets through 16% of the non-matching molecules FragFp alone
216
+ lets through, and 2% for rare queries; no true match was rejected over 23 M true pairs. A screen
217
+ costs about 0.7 times a fingerprint.
218
+
219
+ `getQueryIndex` and `getQueryScreen` read a query the way the substructure search matches it.
220
+ OpenChemLib's own query fingerprint does not yet, and rejects true matches of a query with an
221
+ exclude group cutting an aromatic ring or with a "match formal bond order" bond
222
+ ([Actelion/openchemlib#177](https://github.com/Actelion/openchemlib/issues/177)); it also skips atom
223
+ ring sizes when it matches a key in a query
224
+ ([#178](https://github.com/Actelion/openchemlib/issues/178)). For any other query, `getQueryIndex`
225
+ equals OpenChemLib's own.
226
+
227
+ `getFragFpKeys` and `getScreenKeys` return the two dictionaries as idcodes, in bit order, so a page
228
+ can draw what each bit stands for. Bit i is key i, in word `i >> 5` at `1 << (31 - (i & 31))`:
229
+
230
+ ```js
231
+ const screen = getScreen(idCode);
232
+ const held = getScreenKeys().filter(
233
+ (key, i) => screen[i >> 5] & (1 << (31 - (i & 31))),
234
+ );
235
+ ```
236
+
237
+ ## Function keys
238
+
239
+ `getFunction` adds 512 named bits: bit i is set when the molecule holds function key i — a
240
+ thioether, an aromatic ring, a uracil, a Boc group. A question in words becomes bit arithmetic over
241
+ stored words, and each bit is exactly what `substructureSearch(key.fragments, entries)` answers.
242
+
243
+ The dictionary sits in a subpath, so a page that only searches never loads its 3 MB:
244
+
245
+ ```js
246
+ import {
247
+ getFunctionKey,
248
+ getRecordBit,
249
+ searchFunctionKeys,
250
+ } from 'openchemlib-search-wasm/function-keys';
251
+
252
+ searchFunctionKeys('lactam'); // keys whose names hold "lactam", an exact name first
253
+ const key = getFunctionKey('thioether'); // by id, name or synonym, case aside
254
+ const { word, mask } = getRecordBit(key);
255
+ const holds = (records.words[48 * i + word] & mask) !== 0; // records of getMoleculeRecords
256
+ ```
257
+
258
+ It holds 3,611 keys in 19 categories (`getFunctionCategories`), each with a unique name and its
259
+ synonyms:
260
+
261
+ | tier | keys | answered by |
262
+ | --------------- | ----: | ------------------------------------------------------------------------- |
263
+ | `stored` | 512 | bit `key.plane` of `getFunction`, words 32 to 47 of a record |
264
+ | `existingPlane` | 22 | bit `key.bit` of the FragFp or the screen, which holds the same molecules |
265
+ | `onDemand` | 3,077 | `substructureSearch(key.fragments, entries)` |
266
+
267
+ `getRecordBit` is undefined for an on-demand key. `key.prevalence` is the share of 1,034,054 PubChem
268
+ molecules holding the key, which orders the tests of a query from the most selective. The stored
269
+ keys are frozen with the module: changing, adding or reordering one changes every stored record,
270
+ so the 512 only change in a major release.
271
+
272
+ The definitions are ours and those of ChemCaption, AccFG, medchem, RDKit and openchemlib-utils,
273
+ with their licences in `getFunctionKeySources`. Every SMARTS was compiled to OpenChemLib query
274
+ fragments and checked against RDKit on 100,000 molecules; where the two aromaticity models disagree,
275
+ as on pyridones and coumarins, the key follows OpenChemLib's.
276
+
149
277
  ## Identity hashes
150
278
 
151
279
  Two hashes, for the two things "the same compound" usually means. Both are OpenChemLib's own —
@@ -189,6 +317,87 @@ molecule OpenChemLib cannot canonize, gets `NO_HASH` (`0n`).
189
317
  Both take `{ largestFragmentOnly: true }` to strip everything but the largest fragment and
190
318
  neutralize it first, so a salt hashes as its parent structure.
191
319
 
320
+ ### The canonical form itself
321
+
322
+ Each hash is OpenChemLib's 64-bit StrongHasher over a canonical idcode, and that idcode is available
323
+ too — `getNoStereoIdCode` and `getNoStereoTautomerIdCode`, with `…IdCodes` batch forms. They are the
324
+ exact strings the hashes hash, computed by the same code, so the two can never disagree.
325
+
326
+ Prefer the hash for a column that is only ever compared: 8 bytes that an integer index handles,
327
+ against around 25 characters for a no-stereo idcode and roughly twice that for a tautomer one. Reach
328
+ for the idcode when you need the canonical form itself — to show it, to hand it to another
329
+ OpenChemLib, or because a 1-in-2^64 collision is not acceptable for what the column decides.
330
+
331
+ **A tautomer idcode is a key, not a structure.** Parsing one back gives a molecule with every
332
+ tautomeric bond normalized and pi-electron counts in its atom labels, not the compound.
333
+
334
+ ### Stereochemistry and coordinates
335
+
336
+ These functions canonize with the canonizer told to disregard stereochemistry, rather than by
337
+ stripping stereo from the molecule and canonizing that. The distinction matters because the module
338
+ parses idcodes without inventing coordinates: `stripStereoInformation()` turns an implicit
339
+ double-bond configuration into a cross bond, there is no stereo bond to turn without coordinates, and
340
+ the canonizer then assigns a configuration instead of dropping one. Measured over 400 real idcodes,
341
+ the stripping route disagreed with the answer a caller holding a molecule gets on 6 of them — turning
342
+ an unconfigured double bond into E, and flipping a Z one. So a compound drawn without its
343
+ double-bond stereo keyed differently from the same compound drawn with it, which is the one thing
344
+ these keys exist to prevent.
345
+
346
+ Disregarding stereo in the canonizer agrees with the coordinate-bearing answer on all 400 and costs
347
+ nothing. Inventing coordinates would also agree, and costs 19 times the parse.
348
+
349
+ **This changed the hashes in 2.0.0.** A column of `getNoStereoHash` or `getNoStereoTautomerHash`
350
+ values written by 1.x has to be rebuilt; around 1.5% of them were wrong, and which ones depends on
351
+ whether the molecule had an unconfigured stereogenic double bond.
352
+
353
+ ### Bounding the tautomer tail
354
+
355
+ The cost of a tautomer key is set by how many tautomers the molecule has, not by its size, and the
356
+ spread is enormous: over a drug-like fixture the median molecule takes 0.12 ms and one in a hundred
357
+ takes most of a second. `maxTautomers` is the ceiling OpenChemLib stops at, and lowering it bounds
358
+ that tail. `tautomerCounts` says which molecules ran into it — the only signal that a key came from
359
+ a truncated enumeration, and so may not be canonical:
360
+
361
+ ```js
362
+ const tautomerCounts = new Int32Array(idCodes.length);
363
+ const maxTautomers = 1000;
364
+ const keys = getNoStereoTautomerIdCodes(idCodes, {
365
+ maxTautomers,
366
+ tautomerCounts,
367
+ });
368
+
369
+ // keys[i] is not canonical where this holds — record it next to the key
370
+ const truncated = (i) => tautomerCounts[i] >= maxTautomers;
371
+ ```
372
+
373
+ Measured over 200 fixture idcodes:
374
+
375
+ | `maxTautomers` | median | p99 | slowest | whole batch | truncated |
376
+ | ---------------: | ------: | -----: | ------: | ----------: | --------: |
377
+ | 100000 (default) | 0.12 ms | 807 ms | 969 ms | 1824 ms | 2 |
378
+ | 10000 | 0.06 ms | 46 ms | 63 ms | 135 ms | 2 |
379
+ | 1000 | 0.06 ms | 3 ms | 4 ms | 33 ms | 2 |
380
+ | 100 | 0.06 ms | 1 ms | 1 ms | 17 ms | 8 |
381
+
382
+ At 1000 the slowest molecule is 242x cheaper and the batch 55x, and the same two molecules are
383
+ truncated as at the default — so the ceiling costs nothing on this fixture beyond what the default
384
+ already gave up on. Where it lands for your library is an empirical question: lower it, and count.
385
+
386
+ ### Getting both the key and its hash
387
+
388
+ `strongHash` is OpenChemLib's 64-bit hasher, and every `CanonizerUtil` hash is defined as that hasher
389
+ over a canonical idcode. So a caller that wants both forms asks for the idcode and hashes it, rather
390
+ than calling two functions that each canonize:
391
+
392
+ ```js
393
+ const idCode = getNoStereoTautomerIdCode(entry);
394
+ const hash = strongHash(idCode); // identical to getNoStereoTautomerHash(entry)
395
+ ```
396
+
397
+ That matters because canonizing a generic tautomer is the most expensive thing here: asking for both
398
+ separately costs 9% more on the median molecule and 48% more averaged over a library. It is also how
399
+ a database backfills a hash column from idcodes it already stores, with no chemistry at all.
400
+
192
401
  ### What they cost
193
402
 
194
403
  On ordinary drug-like molecules the no-stereo hash costs about 52 µs and the tautomer one about
@@ -205,6 +414,80 @@ Two things follow. Hash a library **in a worker**, never on the main thread. And
205
414
  identity up to stereochemistry, `getNoStereoHash` is the one to reach for: it is the hash whose cost
206
415
  you can predict.
207
416
 
417
+ ### Getting a hash out of a process
418
+
419
+ A hash is a signed 64-bit integer. `JSON.stringify` throws on a `BigInt`, and a JSON number could
420
+ not carry one anyway, so a hash that leaves the process goes as the 16 hex digits of its pattern:
421
+
422
+ ```js
423
+ import { hashToHex, hexToHash } from 'openchemlib-search-wasm';
424
+
425
+ hashToHex(getNoStereoTautomerHash(idCode)); // 'eb14b9bc9ab4dd52', always 16 characters
426
+ hexToHash('eb14b9bc9ab4dd52'); // -1507375755561738926n
427
+ ```
428
+
429
+ `hash.toString(16)` is not the same thing: it writes `-1` for `-1n`, where the pattern is
430
+ `ffffffffffffffff`, and it pads nothing.
431
+
432
+ Store the hash in an integer column — 8 bytes, and the column indexes as an integer — and let the
433
+ database format it on the way out. SQLite's `printf('%016x', hash)` produces exactly the same
434
+ string, which also keeps a 64-bit value from ever being read into a JavaScript number:
435
+
436
+ ```sql
437
+ SELECT printf('%016x', no_stereo_tautomer_hash) AS hash FROM molecules WHERE id = ?;
438
+ ```
439
+
440
+ ## Molecule records
441
+
442
+ `getMoleculeRecords` computes what a molecule table stores, parsing each idcode once:
443
+
444
+ | field | per molecule |
445
+ | ---------------- | ---------------------------------------------------------------------------------- |
446
+ | `idCodes` | the canonical idcode, as `openchemlib`'s `Molecule.fromIDCode(idCode).getIDCode()` |
447
+ | `words` | 48 words at `48 * i`: the FragFp, the screen, then the function bits |
448
+ | `atomCounts` | `getAllAtoms()`, explicit hydrogens included |
449
+ | `weights` | OpenChemLib's relative weight |
450
+ | `noStereoHashes` | the hash of `getNoStereoHashes` |
451
+ | `tautomerHashes` | the hash of `getNoStereoTautomerHashes`, unless that one is not canonical |
452
+ | `tautomerCounts` | how many tautomers were enumerated |
453
+
454
+ A molecule whose enumeration reaches `maxTautomers` (default `DEFAULT_MAX_TAUTOMERS`) gets its
455
+ no-stereo hash as its tautomer hash, because a key from a cut-short enumeration depends on where it
456
+ stopped. An idcode that will not parse gets `''`, zeros, 0 atoms, `NaN`, `NO_HASH` and a count of 0.
457
+
458
+ The canonical idcode keeps stereochemistry, so it is canonized on invented 2D coordinates, as
459
+ `openchemlib` parses by default. Canonizing the coordinate-free molecule instead gives another
460
+ idcode for 330 of the 1,999 test idcodes.
461
+
462
+ One call costs about what `getIndexes`, `getScreens`, `getFunctions` and the two hash functions cost together: the
463
+ parses it saves roughly pay for the atom count and the weight, and the canonical idcode adds a
464
+ coordinate invention per molecule. `node benchmark/records.js` measures it.
465
+
466
+ `getMoleculeSizes` returns only `atomCounts` and `weights`, for one parse each. `getQueryAtomFloor`
467
+ is the atom count below which a molecule cannot contain a query: its atoms once exclude groups are
468
+ removed, plain hydrogens aside.
469
+
470
+ ## Diagnostics
471
+
472
+ **Nothing here writes to your console.** The generated TeaVM runtime maps Java's standard output
473
+ onto `console.log`, which is the stream a server writing structured logs is already using. Anything
474
+ OpenChemLib printed would interleave unparsable lines into that log, so Java's two streams are
475
+ routed to a sink that drops them, and a host that wants them says where:
476
+
477
+ ```js
478
+ import { setLogHandler } from 'openchemlib-search-wasm';
479
+
480
+ setLogHandler((message, stream) => logger.debug({ stream }, message));
481
+ setLogHandler(null); // back to discarding
482
+ ```
483
+
484
+ The handler applies whether or not the module has been instantiated, so it can be set at startup or
485
+ around a single call. It is per module instance, which means per worker.
486
+
487
+ OpenChemLib's one message, `Tautomer count exceeds maximum`, is switched off at its source rather
488
+ than merely dropped here: it built its text with a full extra canonization of the molecule, once per
489
+ molecule it gave up on. `tautomerCounts` says the same thing without the work.
490
+
208
491
  ## What you actually win
209
492
 
210
493
  Against `openchemlib` 9.25.0, on 409,686 real idcodes, with identical hit counts for every query:
@@ -236,4 +519,8 @@ anything is parsed, which is worth far more than 2x. Search the survivors, not t
236
519
 
237
520
  ## License
238
521
 
239
- BSD-3-Clause.
522
+ BSD-3-Clause. The function-key dictionary adapts definitions from
523
+ [ChemCaption](https://github.com/lamalab-org/chem-caption) (MIT),
524
+ [AccFG](https://github.com/xuanliugit/AccFG) (MIT), [medchem](https://github.com/datamol-io/medchem)
525
+ (Apache-2.0), [RDKit](https://github.com/rdkit/rdkit) (BSD-3-Clause) and
526
+ [openchemlib-utils](https://github.com/cheminfo/openchemlib-utils) (MIT).