@nshiab/simple-data-analysis 6.0.0 → 6.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +128 -11
- package/esm/class/SimpleDB.d.ts +1 -0
- package/esm/class/SimpleDB.d.ts.map +1 -1
- package/esm/class/SimpleDB.js +1 -0
- package/esm/class/SimpleTable.d.ts +75 -64
- package/esm/class/SimpleTable.d.ts.map +1 -1
- package/esm/class/SimpleTable.js +75 -64
- package/llm.md +150 -124
- package/package.json +1 -1
- package/script/class/SimpleDB.d.ts +1 -0
- package/script/class/SimpleDB.d.ts.map +1 -1
- package/script/class/SimpleDB.js +1 -0
- package/script/class/SimpleTable.d.ts +75 -64
- package/script/class/SimpleTable.d.ts.map +1 -1
- package/script/class/SimpleTable.js +75 -64
package/README.md
CHANGED
|
@@ -217,6 +217,7 @@ const data = await sdb
|
|
|
217
217
|
.log();
|
|
218
218
|
|
|
219
219
|
await data.writeData("sda/output/averageTemperatures.csv");
|
|
220
|
+
|
|
220
221
|
await sdb.close();
|
|
221
222
|
```
|
|
222
223
|
|
|
@@ -473,12 +474,115 @@ await sdb.close();
|
|
|
473
474
|
|
|
474
475
|

|
|
475
476
|
|
|
477
|
+
### Google Sheets
|
|
478
|
+
|
|
479
|
+
The
|
|
480
|
+
[`toSheet`](https://jsr.io/@nshiab/simple-data-analysis/doc/~/SimpleTable.prototype.toSheet)
|
|
481
|
+
method sends a table directly to Google Sheets. Authenticate with a service
|
|
482
|
+
account by setting its email and private key in `.env`:
|
|
483
|
+
|
|
484
|
+
```dotenv
|
|
485
|
+
GOOGLE_SERVICE_ACCOUNT_EMAIL=service-account@example.iam.gserviceaccount.com
|
|
486
|
+
GOOGLE_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----\n"
|
|
487
|
+
```
|
|
488
|
+
|
|
489
|
+
Alternatively, point `GOOGLE_APPLICATION_CREDENTIALS` to the service-account
|
|
490
|
+
JSON file:
|
|
491
|
+
|
|
492
|
+
```dotenv
|
|
493
|
+
GOOGLE_APPLICATION_CREDENTIALS=./service-account.json
|
|
494
|
+
```
|
|
495
|
+
|
|
496
|
+
Share the destination spreadsheet with the service-account email before running
|
|
497
|
+
the example. The `loadSheet()` method uses the same environment variables.
|
|
498
|
+
|
|
499
|
+
```ts
|
|
500
|
+
// Uses Google service-account credentials from .env.
|
|
501
|
+
import { SimpleDB } from "@nshiab/simple-data-analysis";
|
|
502
|
+
|
|
503
|
+
const sdb = new SimpleDB();
|
|
504
|
+
const temperatures = sdb.newTable("temperatures");
|
|
505
|
+
|
|
506
|
+
await temperatures
|
|
507
|
+
.loadData(
|
|
508
|
+
"https://raw.githubusercontent.com/nshiab/simple-data-analysis/main/test/data/files/dailyTemperatures.csv",
|
|
509
|
+
)
|
|
510
|
+
.renameColumns({ t: "temperature", id: "station" })
|
|
511
|
+
.selectColumns(["station", "time", "temperature"])
|
|
512
|
+
.toSheet("https://docs.google.com/spreadsheets/d/.../edit#gid=0");
|
|
513
|
+
|
|
514
|
+
await sdb.close();
|
|
515
|
+
```
|
|
516
|
+
|
|
517
|
+
### Datawrapper
|
|
518
|
+
|
|
519
|
+
The
|
|
520
|
+
[`toDatawrapper`](https://jsr.io/@nshiab/simple-data-analysis/doc/~/SimpleTable.prototype.toDatawrapper)
|
|
521
|
+
method sends a table directly to a Datawrapper chart or table. Add your API key
|
|
522
|
+
to `.env`:
|
|
523
|
+
|
|
524
|
+
```dotenv
|
|
525
|
+
DATAWRAPPER_KEY=your-datawrapper-api-key
|
|
526
|
+
```
|
|
527
|
+
|
|
528
|
+
The chart ID is the short identifier in its Datawrapper URL. For maps, use
|
|
529
|
+
[`toGeoDatawrapper`](https://jsr.io/@nshiab/simple-data-analysis/doc/~/SimpleTable.prototype.toGeoDatawrapper)
|
|
530
|
+
with the same API key. The `loadDatawrapper()` and `loadGeoDatawrapper()`
|
|
531
|
+
methods also use it.
|
|
532
|
+
|
|
533
|
+
```ts
|
|
534
|
+
// Uses DATAWRAPPER_KEY from .env.
|
|
535
|
+
import { SimpleDB } from "@nshiab/simple-data-analysis";
|
|
536
|
+
|
|
537
|
+
const sdb = new SimpleDB();
|
|
538
|
+
const temperatures = sdb.newTable("temperatures");
|
|
539
|
+
|
|
540
|
+
await temperatures
|
|
541
|
+
.loadData(
|
|
542
|
+
"https://raw.githubusercontent.com/nshiab/simple-data-analysis/main/test/data/files/dailyTemperatures.csv",
|
|
543
|
+
)
|
|
544
|
+
.renameColumns({ t: "temperature", id: "station" })
|
|
545
|
+
.selectColumns(["station", "time", "temperature"])
|
|
546
|
+
.toDatawrapper("myChartId", { republish: true });
|
|
547
|
+
|
|
548
|
+
await sdb.close();
|
|
549
|
+
```
|
|
550
|
+
|
|
476
551
|
### AI
|
|
477
552
|
|
|
478
553
|
SDA can use LLMs and embedding models to enrich data, search text, and answer
|
|
479
|
-
questions based on the contents of a table.
|
|
480
|
-
|
|
481
|
-
|
|
554
|
+
questions based on the contents of a table. Choose one of the following `.env`
|
|
555
|
+
configurations.
|
|
556
|
+
|
|
557
|
+
For the Gemini API:
|
|
558
|
+
|
|
559
|
+
```dotenv
|
|
560
|
+
AI_PROVIDER=gemini
|
|
561
|
+
AI_MODEL=gemini-3-flash-preview
|
|
562
|
+
AI_EMBEDDINGS_PROVIDER=gemini
|
|
563
|
+
AI_EMBEDDINGS_MODEL=gemini-embedding-001
|
|
564
|
+
AI_KEY=your-gemini-api-key
|
|
565
|
+
```
|
|
566
|
+
|
|
567
|
+
For Vertex AI, replace `AI_KEY` with your Google Cloud project and location:
|
|
568
|
+
|
|
569
|
+
```dotenv
|
|
570
|
+
AI_PROVIDER=gemini
|
|
571
|
+
AI_MODEL=gemini-3-flash-preview
|
|
572
|
+
AI_EMBEDDINGS_PROVIDER=gemini
|
|
573
|
+
AI_EMBEDDINGS_MODEL=gemini-embedding-001
|
|
574
|
+
AI_PROJECT=my-google-cloud-project
|
|
575
|
+
AI_LOCATION=us-central1
|
|
576
|
+
```
|
|
577
|
+
|
|
578
|
+
For local Ollama models:
|
|
579
|
+
|
|
580
|
+
```dotenv
|
|
581
|
+
AI_PROVIDER=ollama
|
|
582
|
+
AI_MODEL=gemma3:4b
|
|
583
|
+
AI_EMBEDDINGS_PROVIDER=ollama
|
|
584
|
+
AI_EMBEDDINGS_MODEL=nomic-embed-text
|
|
585
|
+
```
|
|
482
586
|
|
|
483
587
|
SDA's AI capabilities come from
|
|
484
588
|
[`journalism-ai`](https://jsr.io/@nshiab/journalism-ai). By default, LLM
|
|
@@ -495,11 +599,13 @@ record row-level errors, making it useful for cleaning, extracting, classifying,
|
|
|
495
599
|
and enriching data at scale.
|
|
496
600
|
|
|
497
601
|
```ts
|
|
602
|
+
// Uses AI_PROVIDER, AI_MODEL, and any required credentials from .env.
|
|
498
603
|
import { SimpleDB } from "@nshiab/simple-data-analysis";
|
|
499
604
|
|
|
500
605
|
const sdb = new SimpleDB();
|
|
501
|
-
const cities =
|
|
502
|
-
|
|
606
|
+
const cities = sdb.newTable("cities");
|
|
607
|
+
|
|
608
|
+
await cities
|
|
503
609
|
.loadArray([
|
|
504
610
|
{ city: "Marrakech" },
|
|
505
611
|
{ city: "Kyoto" },
|
|
@@ -509,7 +615,7 @@ const cities = await sdb
|
|
|
509
615
|
"city",
|
|
510
616
|
["country", "continent"],
|
|
511
617
|
"Give me the country and continent of the city.",
|
|
512
|
-
{ concurrency: 5, errorColumn: "error" },
|
|
618
|
+
{ concurrency: 5, errorColumn: "error", verbose: true },
|
|
513
619
|
)
|
|
514
620
|
.log();
|
|
515
621
|
|
|
@@ -529,12 +635,15 @@ and
|
|
|
529
635
|
methods used by `hybridSearch` are also available directly.
|
|
530
636
|
|
|
531
637
|
```ts
|
|
638
|
+
// Uses AI_EMBEDDINGS_PROVIDER, AI_EMBEDDINGS_MODEL, and any required credentials
|
|
639
|
+
// from .env.
|
|
532
640
|
import { SimpleDB } from "@nshiab/simple-data-analysis";
|
|
533
641
|
|
|
534
642
|
const sdb = new SimpleDB();
|
|
643
|
+
const recipes = sdb.newTable("recipes");
|
|
644
|
+
|
|
535
645
|
// We search both the meaning and the wording of each recipe.
|
|
536
|
-
|
|
537
|
-
.newTable("recipes")
|
|
646
|
+
await recipes
|
|
538
647
|
.loadData(
|
|
539
648
|
"https://raw.githubusercontent.com/nshiab/simple-data-analysis/main/test/data/files/recipesClean.parquet",
|
|
540
649
|
)
|
|
@@ -543,9 +652,10 @@ const results = await sdb
|
|
|
543
652
|
"Dish",
|
|
544
653
|
"Recipe",
|
|
545
654
|
5,
|
|
546
|
-
{ outputTable: "results" },
|
|
655
|
+
{ outputTable: "results", verbose: true },
|
|
547
656
|
)
|
|
548
657
|
.log(); // For example: "Butter Pie" (keyword) and "Croissant" (semantic).
|
|
658
|
+
|
|
549
659
|
await sdb.close();
|
|
550
660
|
```
|
|
551
661
|
|
|
@@ -557,6 +667,7 @@ method first retrieves relevant rows with hybrid search, then asks an LLM to
|
|
|
557
667
|
answer using only those rows.
|
|
558
668
|
|
|
559
669
|
```ts
|
|
670
|
+
// Uses both AI provider/model pairs and any required credentials from .env.
|
|
560
671
|
import { SimpleDB } from "@nshiab/simple-data-analysis";
|
|
561
672
|
|
|
562
673
|
const sdb = new SimpleDB();
|
|
@@ -572,9 +683,11 @@ const answer = await sdb
|
|
|
572
683
|
"Dish",
|
|
573
684
|
"Recipe",
|
|
574
685
|
10,
|
|
686
|
+
{ verbose: true },
|
|
575
687
|
);
|
|
576
688
|
|
|
577
689
|
console.log(answer);
|
|
690
|
+
|
|
578
691
|
await sdb.close();
|
|
579
692
|
```
|
|
580
693
|
|
|
@@ -586,19 +699,23 @@ method turns a natural-language instruction into a SQL query and executes it on
|
|
|
586
699
|
the table.
|
|
587
700
|
|
|
588
701
|
```ts
|
|
702
|
+
// Uses AI_PROVIDER, AI_MODEL, and any required credentials from .env.
|
|
589
703
|
import { SimpleDB } from "@nshiab/simple-data-analysis";
|
|
590
704
|
|
|
591
705
|
const sdb = new SimpleDB();
|
|
592
|
-
const
|
|
593
|
-
|
|
706
|
+
const temperatures = sdb.newTable("temperatures");
|
|
707
|
+
|
|
708
|
+
await temperatures
|
|
594
709
|
.loadData(
|
|
595
710
|
"https://raw.githubusercontent.com/nshiab/simple-data-analysis/main/test/data/files/dailyTemperatures.csv",
|
|
596
711
|
)
|
|
597
712
|
.renameColumns({ t: "temperature", id: "station" })
|
|
598
713
|
.aiQuery(
|
|
599
714
|
"Compute the average temperature for each station with two decimals.",
|
|
715
|
+
{ verbose: true },
|
|
600
716
|
)
|
|
601
717
|
.log();
|
|
718
|
+
|
|
602
719
|
await sdb.close();
|
|
603
720
|
```
|
|
604
721
|
|
package/esm/class/SimpleDB.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SimpleDB.d.ts","sourceRoot":"","sources":["../../src/class/SimpleDB.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,qEAAqE,CAAC;AAC/G,OAAO,WAAW,MAAM,kBAAkB,CAAC;AAE3C
|
|
1
|
+
{"version":3,"file":"SimpleDB.d.ts","sourceRoot":"","sources":["../../src/class/SimpleDB.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,qEAAqE,CAAC;AAC/G,OAAO,WAAW,MAAM,kBAAkB,CAAC;AAE3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,MAAM,CAAC,OAAO,OAAO,QAAS,SAAQ,YAAY,CAAC,WAAW,CAAC;IAC7D;;;;OAIG;IACM,UAAU,EAAE,OAAO,WAAW,CAAC;IAExC;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;gBAED,OAAO,GAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,WAAW,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;QAC7B,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KACb;CAMT"}
|