@kineviz/ladybug-lite 0.15.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +84 -0
- package/connection.js +471 -0
- package/database.js +211 -0
- package/index.js +19 -0
- package/index.mjs +10 -0
- package/lbug.d.ts +399 -0
- package/lbug_native.js +25 -0
- package/package.json +27 -0
- package/prepared_statement.js +42 -0
- package/query_result.js +242 -0
- package/util/__pycache__/safe_ladybug_subprocess.cpython-314.pyc +0 -0
- package/util/build.js +248 -0
- package/util/build.md +90 -0
- package/util/buildLadybugExtensions.sh +79 -0
- package/util/buildLadybugWithDocker.sh +65 -0
- package/util/copy.js +24 -0
- package/util/dev.md +419 -0
- package/util/install.js +81 -0
- package/util/readme.md +23 -0
- package/util/safe_ladybug_subprocess.py +58 -0
- package/util/test.Ipynb +120 -0
- package/util/test.js +50 -0
- package/util/test.large.js +26 -0
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
APP_ROOT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}")/..; pwd)
|
|
3
|
+
|
|
4
|
+
SYSTEM="$(uname -o)"
|
|
5
|
+
if [ $SYSTEM == "Msys" ]
|
|
6
|
+
then
|
|
7
|
+
export MSYS2_ARG_CONV_EXCL="*"
|
|
8
|
+
APP_ROOT_DIR="$(cygpath -w $APP_ROOT_DIR)"
|
|
9
|
+
echo "Msys"
|
|
10
|
+
fi
|
|
11
|
+
|
|
12
|
+
# Build amd64 artifact
|
|
13
|
+
docker buildx build --platform=linux/amd64 --target build-amd64 -t ladybug-lite:build-amd64 -f ${APP_ROOT_DIR}/Dockerfile ./
|
|
14
|
+
|
|
15
|
+
# Extract amd64 prebuilt file
|
|
16
|
+
docker create --name extract-amd64 ladybug-lite:build-amd64
|
|
17
|
+
docker cp extract-amd64:/app/node_modules/@ladybugdb/core/prebuilt/lbugjs-alpine-amd64.node ./prebuilt/
|
|
18
|
+
docker rm extract-amd64
|
|
19
|
+
|
|
20
|
+
Build arm64 artifact
|
|
21
|
+
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
|
|
22
|
+
|
|
23
|
+
if ! docker buildx inspect myarmbuilder &>/dev/null; then
|
|
24
|
+
docker buildx create --name myarmbuilder --platform linux/arm64 --use
|
|
25
|
+
else
|
|
26
|
+
docker buildx use myarmbuilder
|
|
27
|
+
fi
|
|
28
|
+
docker buildx build --platform=linux/arm64 --target build-arm64 -t ladybug-lite:build-arm64 -f ${APP_ROOT_DIR}/Dockerfile ./
|
|
29
|
+
|
|
30
|
+
# Extract arm64 prebuilt file
|
|
31
|
+
docker create --name extract-arm64 ladybug-lite:build-arm64
|
|
32
|
+
docker cp extract-arm64:/app/node_modules/@ladybugdb/core/prebuilt/lbugjs-alpine-arm64.node ./prebuilt/
|
|
33
|
+
docker rm extract-arm64
|
|
34
|
+
|
|
35
|
+
echo "All prebuilt files have been extracted to ./prebuilt/"
|
|
36
|
+
|
|
37
|
+
run_test() {
|
|
38
|
+
local arch=$1
|
|
39
|
+
local node_file=$2
|
|
40
|
+
|
|
41
|
+
echo "Running tests on $arch architecture..."
|
|
42
|
+
docker run --rm \
|
|
43
|
+
--platform linux/$arch \
|
|
44
|
+
-v "${APP_ROOT_DIR}:/ladybug-lite" \
|
|
45
|
+
node:18-alpine /bin/sh -c "
|
|
46
|
+
cd /ladybug-lite && \
|
|
47
|
+
cp -f /ladybug-lite/prebuilt/$node_file /ladybug-lite/lbugjs.node && \
|
|
48
|
+
cd /ladybug-lite/util && \
|
|
49
|
+
node ./test.js
|
|
50
|
+
"
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
build_extension(){
|
|
54
|
+
docker run -it -d \
|
|
55
|
+
ladybug-lite:build-amd64 node
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# Run tests for different architectures
|
|
60
|
+
# run_test "amd64" "lbugjs-alpine-amd64.node"
|
|
61
|
+
run_test "arm64" "lbugjs-alpine-arm64.node"
|
|
62
|
+
|
|
63
|
+
echo "All tests completed successfully!"
|
|
64
|
+
|
|
65
|
+
echo "Please use 'yarn release' to release the package to npm!"
|
package/util/copy.js
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
const path = require("path");
|
|
2
|
+
const process = require("process");
|
|
3
|
+
const fs = require("fs");
|
|
4
|
+
const https = require("https");
|
|
5
|
+
const { HttpsProxyAgent } = require('https-proxy-agent');
|
|
6
|
+
const packageJson = require("./../package.json");
|
|
7
|
+
|
|
8
|
+
const rootDir = path.join(__dirname, "..");
|
|
9
|
+
|
|
10
|
+
let arch = process.arch;
|
|
11
|
+
let platform = process.platform;
|
|
12
|
+
const isAlpine = platform == "linux" && fs.readFileSync('/etc/os-release', 'utf8').includes('Alpine Linux');
|
|
13
|
+
if (isAlpine) {
|
|
14
|
+
platform = "alpine";
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
if(arch == "x64"){
|
|
18
|
+
arch="amd64"
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const prebuiltURL = `prebuilt/lbugjs-${platform}-${arch}.node`;
|
|
22
|
+
|
|
23
|
+
// force copy prebuilt file to ./lbugjs.node
|
|
24
|
+
fs.copyFileSync(path.join(rootDir, prebuiltURL), path.join(rootDir, "lbugjs.node"));
|
package/util/dev.md
ADDED
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
# ladybug-lite — Developer Guide
|
|
2
|
+
|
|
3
|
+
> This document is the single entry point for contributors. It supersedes the
|
|
4
|
+
> scattered notes in [`util/build.md`](build.md) and [`util/readme.md`](readme.md),
|
|
5
|
+
> which are kept as historical references.
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## 1. What this repository is
|
|
10
|
+
|
|
11
|
+
`@kineviz/ladybug-lite` is a **redistribution layer** on top of the upstream
|
|
12
|
+
[`@ladybugdb/core`](https://www.npmjs.com/package/@ladybugdb/core) npm package.
|
|
13
|
+
It exists for two reasons:
|
|
14
|
+
|
|
15
|
+
1. **Smaller install footprint.** The upstream package ships every prebuilt
|
|
16
|
+
binary for every platform in one tarball. We only ship the loader/JS and
|
|
17
|
+
download the matching native binary on demand at install time.
|
|
18
|
+
2. **Alpine / musl libc support.** Upstream does not publish Alpine binaries.
|
|
19
|
+
We build them ourselves (via `docker buildx` + QEMU) and host them in this
|
|
20
|
+
repository's `prebuilt/` folder.
|
|
21
|
+
|
|
22
|
+
We do **not** fork the C++ engine. The runtime JS files
|
|
23
|
+
([`connection.js`](../connection.js), [`database.js`](../database.js),
|
|
24
|
+
[`query_result.js`](../query_result.js),
|
|
25
|
+
[`prepared_statement.js`](../prepared_statement.js),
|
|
26
|
+
[`lbug_native.js`](../lbug_native.js), [`index.js`](../index.js),
|
|
27
|
+
[`index.mjs`](../index.mjs), [`lbug.d.ts`](../lbug.d.ts)) are copied verbatim
|
|
28
|
+
from `@ladybugdb/core` by [`util/build.js`](build.js) on each release.
|
|
29
|
+
|
|
30
|
+
---
|
|
31
|
+
|
|
32
|
+
## 2. End-to-end data flow
|
|
33
|
+
|
|
34
|
+
```
|
|
35
|
+
upstream npm
|
|
36
|
+
┌────────────────────────┐
|
|
37
|
+
│ @ladybugdb/core@X.Y.Z │ (yarn add --force)
|
|
38
|
+
└───────────┬────────────┘
|
|
39
|
+
│
|
|
40
|
+
▼
|
|
41
|
+
┌─────────────────────────┐
|
|
42
|
+
│ node_modules/@ladybugdb │
|
|
43
|
+
│ /core/ │
|
|
44
|
+
│ ├─ *.js (runtime API) │
|
|
45
|
+
│ ├─ lbug-source/ │ C++ sources (build only)
|
|
46
|
+
│ └─ prebuilt/*.node │ upstream's prebuilt binaries
|
|
47
|
+
└───────────┬─────────────┘
|
|
48
|
+
│
|
|
49
|
+
┌────────────┼─────────────────────────┐
|
|
50
|
+
│ │ │
|
|
51
|
+
▼ ▼ ▼
|
|
52
|
+
util/build.js buildLadybugWithDocker.sh buildLadybugExtensions.sh
|
|
53
|
+
(copies JS to (compiles Alpine (compiles extensions:
|
|
54
|
+
repo root, amd64/arm64 .node httpfs, json, fts,
|
|
55
|
+
bumps version, via docker buildx) vector, neo4j, algo)
|
|
56
|
+
npm publish) │ │
|
|
57
|
+
│ ▼ ▼
|
|
58
|
+
│ prebuilt/lbugjs- extensions/alpine-{arch}/
|
|
59
|
+
│ alpine-{arch}.node *.lbug_extension
|
|
60
|
+
│ │ │
|
|
61
|
+
▼ ▼ │
|
|
62
|
+
┌─────────────────────────────────────┐ │
|
|
63
|
+
│ git tag X.Y.Z + push prebuilt/ │◄─────────┘
|
|
64
|
+
└────────────────┬────────────────────┘
|
|
65
|
+
│
|
|
66
|
+
▼
|
|
67
|
+
┌──────────────────┐
|
|
68
|
+
│ npm publish │
|
|
69
|
+
│ (release script)│
|
|
70
|
+
└────────┬─────────┘
|
|
71
|
+
│
|
|
72
|
+
▼
|
|
73
|
+
┌─────────────────────────┐
|
|
74
|
+
│ user: npm install │
|
|
75
|
+
│ @kineviz/ladybug-lite │
|
|
76
|
+
└────────┬────────────────┘
|
|
77
|
+
│
|
|
78
|
+
▼ (postinstall hook)
|
|
79
|
+
util/install.js
|
|
80
|
+
│
|
|
81
|
+
│ detects platform / arch / Alpine
|
|
82
|
+
▼
|
|
83
|
+
https://raw.githubusercontent.com/Kineviz/ladybug-lite/
|
|
84
|
+
refs/tags/{version}/prebuilt/lbugjs-{platform}-{arch}.node
|
|
85
|
+
│ (CDN fallback: graphxr.oss-cn-shanghai.aliyuncs.com)
|
|
86
|
+
▼
|
|
87
|
+
writes ./lbugjs.node ──► loaded at runtime by
|
|
88
|
+
lbug_native.js (RTLD_GLOBAL on Linux)
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
## 3. Repository layout (by responsibility)
|
|
94
|
+
|
|
95
|
+
| Group | Path | Purpose |
|
|
96
|
+
| --- | --- | --- |
|
|
97
|
+
| Runtime API (synced from upstream) | [`index.js`](../index.js), [`index.mjs`](../index.mjs), [`connection.js`](../connection.js), [`database.js`](../database.js), [`query_result.js`](../query_result.js), [`prepared_statement.js`](../prepared_statement.js), [`lbug_native.js`](../lbug_native.js), [`lbug.d.ts`](../lbug.d.ts) | Pure JS surface; do **not** edit by hand — they are overwritten by `build.js` |
|
|
98
|
+
| Native binary (per-machine) | `lbugjs.node` (root) | Active binary loaded at runtime; produced by `install.js` or `copy.js` |
|
|
99
|
+
| Prebuilt binaries (shipped) | [`prebuilt/`](../prebuilt/) | One `lbugjs-{platform}-{arch}.node` per supported target |
|
|
100
|
+
| Build / install scripts | [`util/`](.) | Everything in this guide |
|
|
101
|
+
| CI workflows | [`.github/workflows/`](../.github/workflows/) | `build.yaml`, `buildExtension.yaml`, `daily.yaml` |
|
|
102
|
+
| Container build | [`Dockerfile`](../Dockerfile) | Multi-stage, multi-arch Alpine build (amd64 + arm64) |
|
|
103
|
+
| Packaging | [`package.json`](../package.json), [`.npmignore`](../.npmignore) | The `.npmignore` whitelists `util/*.*` and `package.json` only — everything else is included by default |
|
|
104
|
+
|
|
105
|
+
---
|
|
106
|
+
|
|
107
|
+
## 4. Environment prerequisites
|
|
108
|
+
|
|
109
|
+
| Tool | Why | Required for |
|
|
110
|
+
| --- | --- | --- |
|
|
111
|
+
| Node.js 18 | Build & test | Everything |
|
|
112
|
+
| yarn | Package manager used by all scripts | Everything |
|
|
113
|
+
| Docker (with buildx) | Multi-arch Alpine builds | `build:ladybug`, extensions |
|
|
114
|
+
| `qemu-user-static` (host) | Cross-arch emulation when building arm64 on amd64 host | `buildLadybugWithDocker.sh` |
|
|
115
|
+
| `npm` token in env (`NPM_TOKEN`) | Publishing | `release` |
|
|
116
|
+
| Python 3 + `pandas` (optional) | Only for the Python subprocess helper | `safe_ladybug_subprocess.py` |
|
|
117
|
+
|
|
118
|
+
On Debian/Ubuntu hosts:
|
|
119
|
+
|
|
120
|
+
```sh
|
|
121
|
+
sudo apt-get install -y qemu-user-static
|
|
122
|
+
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
---
|
|
126
|
+
|
|
127
|
+
## 5. `util/` scripts in detail
|
|
128
|
+
|
|
129
|
+
### 5.1 `build.js` — sync upstream + publish
|
|
130
|
+
|
|
131
|
+
Driven by `yarn build` (alias for `node util/build.js`).
|
|
132
|
+
|
|
133
|
+
What it does, in order:
|
|
134
|
+
|
|
135
|
+
1. **`deleteFiles(rootDir, [...])`** — wipes the repo root **except** for a
|
|
136
|
+
small allowlist (`package.json`, `util`, `node_modules`, `README.md`,
|
|
137
|
+
`test`, `.git`, `.vscode`, `.gitignore`, `.github`, `.dockerignore`,
|
|
138
|
+
`Dockerfile`, `.npmignore`, `docs`, `prebuilt`).
|
|
139
|
+
2. **`copyDir(node_modules/@ladybugdb/core, rootDir, [...])`** — copies the
|
|
140
|
+
upstream package into the repo root, **excluding**:
|
|
141
|
+
- `lbug-source` (multi-GB C++ source tree)
|
|
142
|
+
- `node_modules`
|
|
143
|
+
- `lbugjs.node` (we ship per-platform copies separately)
|
|
144
|
+
- upstream `package.json` (we keep ours)
|
|
145
|
+
- upstream `install.js` (replaced by ours)
|
|
146
|
+
- upstream `README.md`, `Dockerfile`, `.npmignore`, etc. (we keep ours)
|
|
147
|
+
- upstream `prebuilt/` (we ship our own from `prebuilt/`)
|
|
148
|
+
3. **`asyncVersion()`** — reads upstream's `package.json` version, mirrors it
|
|
149
|
+
into our `package.json` and `devDependencies["@ladybugdb/core"]`, then
|
|
150
|
+
triggers `npmPublish()`.
|
|
151
|
+
4. **`npmPublish()`** — writes `.npmrc` from `process.env.NPM_TOKEN` and runs
|
|
152
|
+
`npm publish --access public --registry https://registry.npmjs.org`.
|
|
153
|
+
|
|
154
|
+
> **Heads up:** `build.js` is destructive. Always run it on a clean working
|
|
155
|
+
> tree on the `release` branch — never on `main` or a feature branch with
|
|
156
|
+
> uncommitted work. CI runs it after `yarn add @ladybugdb/core --force`, so
|
|
157
|
+
> the upstream is always present in `node_modules/`.
|
|
158
|
+
|
|
159
|
+
### 5.2 `install.js` — runs on the consumer's machine
|
|
160
|
+
|
|
161
|
+
Driven by `npm install` via the `"install"` lifecycle script in
|
|
162
|
+
`package.json`.
|
|
163
|
+
|
|
164
|
+
Detection logic:
|
|
165
|
+
|
|
166
|
+
```
|
|
167
|
+
arch = process.arch // x64, arm64, ...
|
|
168
|
+
platform = process.platform // linux, darwin, win32
|
|
169
|
+
|
|
170
|
+
if platform == "linux" and /etc/os-release contains "Alpine Linux":
|
|
171
|
+
platform = "alpine"
|
|
172
|
+
if arch == "x64": arch = "amd64" // historical naming
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Download:
|
|
176
|
+
|
|
177
|
+
1. Primary: `https://raw.githubusercontent.com/Kineviz/ladybug-lite/refs/tags/{version}/prebuilt/lbugjs-{platform}-{arch}.node`
|
|
178
|
+
2. CDN fallback (on any error): `https://graphxr.oss-cn-shanghai.aliyuncs.com/ladybug@{version}/lbugjs-{platform}-{arch}.node`
|
|
179
|
+
|
|
180
|
+
Both honor `HTTP_PROXY` / `HTTPS_PROXY` / `PROXY` (and lowercase variants) via
|
|
181
|
+
`https-proxy-agent`.
|
|
182
|
+
|
|
183
|
+
Written to `./lbugjs.node` in the package root. The version used in the URL is
|
|
184
|
+
`packageJson.version.split("-")[0]` — so a `0.15.3-beta.1` would still pull
|
|
185
|
+
`0.15.3` artifacts.
|
|
186
|
+
|
|
187
|
+
> **If both downloads fail**, the install error message includes
|
|
188
|
+
> `sean@kineviz.com` as the contact. The most common cause is a brand-new
|
|
189
|
+
> version where CI hasn't yet pushed the prebuilt for the user's platform —
|
|
190
|
+
> see §10.
|
|
191
|
+
|
|
192
|
+
### 5.3 `copy.js` — local shortcut
|
|
193
|
+
|
|
194
|
+
```sh
|
|
195
|
+
yarn copy
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
Picks the matching binary out of `prebuilt/` and copies it to `./lbugjs.node`.
|
|
199
|
+
Use this on a developer machine to test against a freshly committed binary
|
|
200
|
+
without having to publish a new version.
|
|
201
|
+
|
|
202
|
+
### 5.4 `test.js` / `test.large.js`
|
|
203
|
+
|
|
204
|
+
- [`test.js`](test.js) — creates a tiny Movie/Person/ActedIn graph in a
|
|
205
|
+
fresh on-disk DB at `util/demo_test.db`, runs a `MATCH` query, prints rows.
|
|
206
|
+
Used by CI as the smoke test inside Alpine containers.
|
|
207
|
+
- [`test.large.js`](test.large.js) — runs a 20k-row query against an
|
|
208
|
+
existing `util/demo_large` DB. Not used in CI; for local performance
|
|
209
|
+
spot-checks only.
|
|
210
|
+
|
|
211
|
+
### 5.5 `buildLadybugWithDocker.sh`
|
|
212
|
+
|
|
213
|
+
Local-only multi-arch Alpine build (CI uses an inline equivalent in
|
|
214
|
+
`build.yaml`):
|
|
215
|
+
|
|
216
|
+
1. `docker buildx build --platform=linux/amd64 --target build-amd64` — builds
|
|
217
|
+
the `build-amd64` stage from [`Dockerfile`](../Dockerfile).
|
|
218
|
+
2. `docker create` + `docker cp` to extract
|
|
219
|
+
`lbugjs-alpine-amd64.node` into `./prebuilt/`.
|
|
220
|
+
3. Resets QEMU (`multiarch/qemu-user-static --reset -p yes`).
|
|
221
|
+
4. Same dance for `linux/arm64`.
|
|
222
|
+
5. Runs `test.js` inside an `arm64` Alpine container against the freshly
|
|
223
|
+
extracted binary.
|
|
224
|
+
|
|
225
|
+
> **Known papercut:** in the current script, line 20 reads
|
|
226
|
+
> `Build arm64 artifact` (no leading `#`), which the shell will try to
|
|
227
|
+
> execute as a command. It fails harmlessly with "command not found" and
|
|
228
|
+
> the script continues. Add a `#` if the noise bothers you.
|
|
229
|
+
|
|
230
|
+
### 5.6 `buildLadybugExtensions.sh`
|
|
231
|
+
|
|
232
|
+
Independently builds the engine extensions (`httpfs`, `json`, `fts`,
|
|
233
|
+
`vector`, `neo4j`, `algo`) via cmake against
|
|
234
|
+
`node_modules/@ladybugdb/core/lbug-source/`. Must run **inside** the target
|
|
235
|
+
Alpine container (the GitHub workflow `buildExtension.yaml` does exactly
|
|
236
|
+
that).
|
|
237
|
+
|
|
238
|
+
Outputs land in `${LBUG_SOURCE_DIR}/extension/alpine-{arch}/` as
|
|
239
|
+
`*.lbug_extension` and `*.kuzu_extension` files. `buildExtension.yaml`
|
|
240
|
+
commits them under `extensions/` in this repo.
|
|
241
|
+
|
|
242
|
+
> **Note on extension distribution:** the engine resolves extensions at
|
|
243
|
+
> runtime by downloading from
|
|
244
|
+
> `https://extension.ladybugdb.com/v{version}/{platform}/{name}` into
|
|
245
|
+
> `~/.ladybug/extension/{version}/{platform}/`. The extensions we build
|
|
246
|
+
> here are intended for that hosting layer, **not** to be shipped inside
|
|
247
|
+
> the npm tarball.
|
|
248
|
+
|
|
249
|
+
### 5.7 `safe_ladybug_subprocess.py`
|
|
250
|
+
|
|
251
|
+
A standalone Python helper that runs the **`real_ladybug`** Python binding
|
|
252
|
+
(not this npm package) inside a forked subprocess via `multiprocessing.Pipe`,
|
|
253
|
+
so a crash in the engine cannot take down the parent process. It is unrelated
|
|
254
|
+
to the npm publish flow and is included here only for convenience of teams
|
|
255
|
+
who use both bindings. Skip it unless you are working on Python integration.
|
|
256
|
+
|
|
257
|
+
---
|
|
258
|
+
|
|
259
|
+
## 6. Three release paths
|
|
260
|
+
|
|
261
|
+
### 6.1 Local manual release (full)
|
|
262
|
+
|
|
263
|
+
Use this when you need to ship an Alpine update from your laptop, e.g. when
|
|
264
|
+
CI is wedged.
|
|
265
|
+
|
|
266
|
+
```sh
|
|
267
|
+
# 0. Be on the release branch with a clean tree.
|
|
268
|
+
git checkout release && git status
|
|
269
|
+
|
|
270
|
+
# 1. Pull the upstream version you want to track.
|
|
271
|
+
yarn cache clean
|
|
272
|
+
yarn add @ladybugdb/core --force
|
|
273
|
+
|
|
274
|
+
# 2. Build Alpine prebuilts (amd64 + arm64) into ./prebuilt/.
|
|
275
|
+
yarn build:ladybug # = bash util/buildLadybugWithDocker.sh
|
|
276
|
+
|
|
277
|
+
# 3. (Optional) Build native prebuilts for the host platform if missing.
|
|
278
|
+
# For Linux x64 / Linux arm64 / darwin-arm64 / win32-x64 you typically
|
|
279
|
+
# rely on the corresponding GitHub Actions runner; do this manually
|
|
280
|
+
# only if you have the right hardware.
|
|
281
|
+
|
|
282
|
+
# 4. Sync upstream JS into the repo root, bump version, and publish.
|
|
283
|
+
NPM_TOKEN=xxxxx yarn build # runs util/build.js → npm publish
|
|
284
|
+
|
|
285
|
+
# 5. Tag the release so install.js can fetch from raw.githubusercontent.
|
|
286
|
+
git tag -a "$(node -p "require('./package.json').version")" -m "Release"
|
|
287
|
+
git push origin --tags
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
Verify the consumer flow:
|
|
291
|
+
|
|
292
|
+
```sh
|
|
293
|
+
mkdir /tmp/verify && cd /tmp/verify && npm init -y
|
|
294
|
+
npm install @kineviz/ladybug-lite
|
|
295
|
+
node -e "console.log(require('@kineviz/ladybug-lite').VERSION)"
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
### 6.2 CI-driven release (the normal path)
|
|
299
|
+
|
|
300
|
+
There are three workflows in [`.github/workflows/`](../.github/workflows/):
|
|
301
|
+
|
|
302
|
+
| Workflow | Trigger | What it does |
|
|
303
|
+
| --- | --- | --- |
|
|
304
|
+
| [`build.yaml`](../.github/workflows/build.yaml) | push / PR to `release`, manual | Matrix `{amd64, arm64}` on Ubuntu runners. Builds Alpine `.node` inside `node:18-alpine` container, copies to `prebuilt/`, runs `util/test.js` smoke test in Alpine, commits `prebuilt/*.node + *.js + package.json` to the release branch, tags with the upstream version (`0.15.3` etc.), publishes via `yarn build`. |
|
|
305
|
+
| [`buildMacOsIntel.yaml`](../.github/workflows/buildMacOsIntel.yaml) | push / PR to `release`, manual | Single job on `macos-26-intel`. Compiles `lbug-source/tools/nodejs_api` from source (upstream ships no darwin-x64), copies the result to `prebuilt/lbugjs-darwin-x64.node`, runs `util/test.js` natively, commits/tags/publishes the same way `build.yaml` does. Races with `build.yaml` on push/tag — the `merge -X ours` strategy and `npm publish` no-op-on-existing handle this safely. |
|
|
306
|
+
| [`buildExtension.yaml`](../.github/workflows/buildExtension.yaml) | push / PR to `release`, manual | Matrix `{amd64, arm64}`. Runs `util/buildLadybugExtensions.sh` inside `node:18-alpine`, force-pushes the resulting `extensions/*/*.lbug_extension` files. |
|
|
307
|
+
| [`daily.yaml`](../.github/workflows/daily.yaml) | cron `0 0 * * *` | Compares `npm view @ladybugdb/core version` vs `npm view @kineviz/ladybug-lite version`. If upstream is newer, opens an issue and triggers `build.yaml` via `workflow_dispatch`. **Note:** does not currently dispatch `buildMacOsIntel.yaml` — Intel Mac binaries piggyback on the next manual or PR-triggered run. |
|
|
308
|
+
|
|
309
|
+
So the **normal release cadence** is: upstream bumps → `daily.yaml` notices →
|
|
310
|
+
`build.yaml` runs on both arches → tag + npm publish. No human action
|
|
311
|
+
required unless something breaks.
|
|
312
|
+
|
|
313
|
+
> **Secrets** required in GitHub `TOKENS` environment: `NPM_TOKEN` (publish),
|
|
314
|
+
> `NPM_TOKENS` (npmrc auth — yes, both spellings are used as of today).
|
|
315
|
+
|
|
316
|
+
### 6.3 Local quick verify (no publish)
|
|
317
|
+
|
|
318
|
+
For day-to-day iteration on the JS surface or `util/` scripts:
|
|
319
|
+
|
|
320
|
+
```sh
|
|
321
|
+
yarn add @ladybugdb/core --force # only if you don't already have it
|
|
322
|
+
yarn copy # prebuilt/* → ./lbugjs.node
|
|
323
|
+
yarn test # runs util/test.js
|
|
324
|
+
```
|
|
325
|
+
|
|
326
|
+
This skips `build.js` entirely — useful when you only need a working
|
|
327
|
+
runtime and want to avoid rewriting the repo root.
|
|
328
|
+
|
|
329
|
+
---
|
|
330
|
+
|
|
331
|
+
## 7. Prebuilt binary naming convention
|
|
332
|
+
|
|
333
|
+
| File | Built by | Notes |
|
|
334
|
+
| --- | --- | --- |
|
|
335
|
+
| `lbugjs-linux-x64.node` | upstream `@ladybugdb/core` | Copied from `node_modules/@ladybugdb/core/prebuilt/` |
|
|
336
|
+
| `lbugjs-linux-arm64.node` | upstream | same |
|
|
337
|
+
| `lbugjs-darwin-arm64.node` | upstream | Apple Silicon |
|
|
338
|
+
| `lbugjs-darwin-x64.node` | **us** (`buildMacOsIntel.yaml`, `macos-26-intel` runner) | Built from source; upstream does not ship Intel Mac for 0.15.x |
|
|
339
|
+
| `lbugjs-win32-x64.node` | upstream | |
|
|
340
|
+
| `lbugjs-alpine-amd64.node` | **us** (`buildLadybugWithDocker.sh` / `build.yaml`) | musl libc |
|
|
341
|
+
| `lbugjs-alpine-arm64.node` | **us** | musl libc |
|
|
342
|
+
|
|
343
|
+
**Time-bounded support:** `darwin-x64` (Intel Mac) depends on the
|
|
344
|
+
`macos-26-intel` GitHub-hosted runner, which Apple/GitHub will retire in
|
|
345
|
+
Fall 2027. After that, Intel Mac users must build from source locally.
|
|
346
|
+
|
|
347
|
+
The naming inconsistency between `linux-x64` (Node arch) and `alpine-amd64`
|
|
348
|
+
(Docker arch) is historical and is normalized by `install.js` — see §5.2.
|
|
349
|
+
|
|
350
|
+
---
|
|
351
|
+
|
|
352
|
+
## 8. Engine extensions
|
|
353
|
+
|
|
354
|
+
When user code runs `LOAD EXTENSION httpfs`, the engine downloads the
|
|
355
|
+
extension from `https://extension.ladybugdb.com/v{VERSION}/{platform}/{ext}`
|
|
356
|
+
into `~/.ladybug/extension/{VERSION}/{platform}/`. We do not need to ship
|
|
357
|
+
extensions inside the npm tarball — but for Alpine targets we **do** need to
|
|
358
|
+
build them ourselves (since upstream's CDN may not host musl variants for
|
|
359
|
+
every release).
|
|
360
|
+
|
|
361
|
+
- Build script: [`util/buildLadybugExtensions.sh`](buildLadybugExtensions.sh)
|
|
362
|
+
- CI: [`.github/workflows/buildExtension.yaml`](../.github/workflows/buildExtension.yaml)
|
|
363
|
+
- Output committed at: `extensions/alpine-{arch}/*.lbug_extension`
|
|
364
|
+
|
|
365
|
+
If you're adding a new extension to the build list, edit the
|
|
366
|
+
`EXTENSION_LIST` variable in `buildLadybugExtensions.sh` (currently
|
|
367
|
+
`httpfs;json;fts;vector;neo4j;algo`).
|
|
368
|
+
|
|
369
|
+
---
|
|
370
|
+
|
|
371
|
+
## 9. Common issues and fixes
|
|
372
|
+
|
|
373
|
+
**`Failed to download: 404` during `npm install`.**
|
|
374
|
+
The tag for your version doesn't exist yet, or the prebuilt for your
|
|
375
|
+
platform/arch wasn't pushed. Check
|
|
376
|
+
`https://github.com/Kineviz/ladybug-lite/tree/{version}/prebuilt`. If the
|
|
377
|
+
file is missing, re-run `build.yaml` for the affected arch.
|
|
378
|
+
|
|
379
|
+
**Both primary and CDN downloads fail behind a corporate proxy.**
|
|
380
|
+
Set `HTTPS_PROXY=http://user:pass@host:port` (case-insensitive variants
|
|
381
|
+
also accepted) before `npm install`. See `install.js:33`.
|
|
382
|
+
|
|
383
|
+
**Alpine container reports `Error loading shared library libc.musl-...`.**
|
|
384
|
+
You loaded a `lbugjs-linux-*.node` (glibc) inside Alpine. Confirm
|
|
385
|
+
`install.js` detected Alpine — it greps `/etc/os-release` for the literal
|
|
386
|
+
string `Alpine Linux`. Distros that derive from Alpine but rewrote that
|
|
387
|
+
file will fail detection.
|
|
388
|
+
|
|
389
|
+
**`docker buildx` complains about `linux/arm64` on an amd64 host.**
|
|
390
|
+
You forgot the QEMU registration step. Run
|
|
391
|
+
`docker run --rm --privileged multiarch/qemu-user-static --reset -p yes`
|
|
392
|
+
once per boot.
|
|
393
|
+
|
|
394
|
+
**`build.js` deleted my work.** Read §5.1 again — it wipes the repo root.
|
|
395
|
+
Always run it on a clean `release` branch.
|
|
396
|
+
|
|
397
|
+
**`yarn build` runs out of memory inside a small Alpine container.**
|
|
398
|
+
Limit C++ build threads. In `node_modules/@ladybugdb/core/lbug-source/tools/nodejs_api/build.js`:
|
|
399
|
+
|
|
400
|
+
```sh
|
|
401
|
+
sed -i 's/THREADS =/THREADS = 2;\/\//' build.js
|
|
402
|
+
```
|
|
403
|
+
|
|
404
|
+
---
|
|
405
|
+
|
|
406
|
+
## 10. Adding a new platform or version — checklist
|
|
407
|
+
|
|
408
|
+
When the supported matrix changes (new Node version, new arch, new platform):
|
|
409
|
+
|
|
410
|
+
- [ ] Add the `lbugjs-{platform}-{arch}.node` entry to `prebuilt/`
|
|
411
|
+
- [ ] Verify `install.js` detection logic produces the right
|
|
412
|
+
`{platform}-{arch}` string for that OS (add a branch if not)
|
|
413
|
+
- [ ] Add a corresponding matrix entry to `.github/workflows/build.yaml`
|
|
414
|
+
(and `buildExtension.yaml` if the extension matrix changes)
|
|
415
|
+
- [ ] Add a build stage to `Dockerfile` if the platform is Alpine-based
|
|
416
|
+
- [ ] Update the compatibility table in the root [`README.md`](../README.md)
|
|
417
|
+
- [ ] Update §7 of this document
|
|
418
|
+
- [ ] Run `yarn copy && yarn test` on a real machine of that platform
|
|
419
|
+
before tagging
|
package/util/install.js
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
const path = require("path");
|
|
2
|
+
const process = require("process");
|
|
3
|
+
const fs = require("fs");
|
|
4
|
+
const https = require("https");
|
|
5
|
+
const { HttpsProxyAgent } = require('https-proxy-agent');
|
|
6
|
+
const packageJson = require("./../package.json");
|
|
7
|
+
|
|
8
|
+
const rootDir = path.join(__dirname, "..");
|
|
9
|
+
|
|
10
|
+
let arch = process.arch;
|
|
11
|
+
let platform = process.platform;
|
|
12
|
+
const isAlpine = platform == "linux" && fs.readFileSync('/etc/os-release', 'utf8').includes('Alpine Linux');
|
|
13
|
+
if (isAlpine) {
|
|
14
|
+
platform = "alpine";
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
if(arch == "x64"){
|
|
18
|
+
arch="amd64"
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const packageVersion = String(packageJson.version).split("-")[0];
|
|
22
|
+
|
|
23
|
+
const baseURL = `https://raw.githubusercontent.com/Kineviz/ladybug-lite/refs/tags/${packageVersion}/prebuilt`;
|
|
24
|
+
const prebuiltURL = `${baseURL}/lbugjs-${platform}-${arch}.node`;
|
|
25
|
+
|
|
26
|
+
console.log(`Downloading prebuilt binary from ${prebuiltURL}...`);
|
|
27
|
+
|
|
28
|
+
const targetPath = path.join(rootDir, "lbugjs.node");
|
|
29
|
+
|
|
30
|
+
const download = (url, dest) => {
|
|
31
|
+
return new Promise((resolve, reject) => {
|
|
32
|
+
const file = fs.createWriteStream(dest);
|
|
33
|
+
const proxy = process.env.HTTP_PROXY || process.env.HTTPS_PROXY || process.env.PROXY || process.env.http_proxy || process.env.https_proxy || process.env.proxy;
|
|
34
|
+
const agent = proxy ? new HttpsProxyAgent(proxy) : undefined;
|
|
35
|
+
|
|
36
|
+
const reqOptions = { agent };
|
|
37
|
+
|
|
38
|
+
https.get(url, reqOptions, (response) => {
|
|
39
|
+
if (response.statusCode !== 200) {
|
|
40
|
+
reject(new Error(`Failed to download: ${response.statusCode} ${response.statusMessage}`));
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
response.pipe(file);
|
|
45
|
+
|
|
46
|
+
file.on('finish', () => {
|
|
47
|
+
file.close(() => resolve());
|
|
48
|
+
});
|
|
49
|
+
}).on('error', (err) => {
|
|
50
|
+
fs.unlink(dest, () => {});
|
|
51
|
+
reject(err);
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
const downloadWithCDN = (error) =>{
|
|
58
|
+
const CDNBaseURL=`https://graphxr.oss-cn-shanghai.aliyuncs.com/ladybug@${packageVersion}`
|
|
59
|
+
const newPrebuiltURL = prebuiltURL.replace(baseURL,CDNBaseURL);
|
|
60
|
+
download(newPrebuiltURL, targetPath)
|
|
61
|
+
.then(() => {
|
|
62
|
+
console.log(`Successfully downloaded to ${targetPath}`);
|
|
63
|
+
console.log("Done!");
|
|
64
|
+
process.exit(0);
|
|
65
|
+
})
|
|
66
|
+
.catch(() => {
|
|
67
|
+
console.error(`Error downloading prebuilt binary: ${error.message}`);
|
|
68
|
+
console.log("Prebuilt binary download failed. Please contact sean@kineviz.com .");
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
download(prebuiltURL, targetPath)
|
|
73
|
+
.then(() => {
|
|
74
|
+
console.log(`Successfully downloaded to ${targetPath}`);
|
|
75
|
+
console.log("Done!");
|
|
76
|
+
process.exit(0);
|
|
77
|
+
})
|
|
78
|
+
.catch((err) => {
|
|
79
|
+
console.log("Try download with CND");
|
|
80
|
+
downloadWithCDN(err)
|
|
81
|
+
});
|
package/util/readme.md
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
## ladybug-lite
|
|
2
|
+
A lightweight fork of the [Ladybug](https://github.com/LadybugDB/ladybug) (formerly Kùzu) embedded graph database, optimized for faster installation and broader compatibility.
|
|
3
|
+
|
|
4
|
+
### Why We Forked Ladybug
|
|
5
|
+
|
|
6
|
+
- ***Large Package Size:***
|
|
7
|
+
The official Ladybug npm package can be sizable, resulting in slow downloads and build times, particularly outside Europe and North America.
|
|
8
|
+
**ladybug-lite** strips it down to essential binaries for a smaller, faster package.
|
|
9
|
+
|
|
10
|
+
- ***No Alpine Linux Support:***
|
|
11
|
+
Ladybug doesn't support Alpine Linux out of the box, which we rely on for Docker image.
|
|
12
|
+
**ladybug-lite** includes musl libc-compatible binaries to work seamlessly with lightweight containers.
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
### Benefits
|
|
16
|
+
|
|
17
|
+
- ***Speed:*** Quicker downloads and builds.
|
|
18
|
+
|
|
19
|
+
- ***Compatibility:*** Full support for Alpine Linux.
|
|
20
|
+
|
|
21
|
+
- ***Efficiency:*** Retains Ladybug's core functionality in a leaner package.
|
|
22
|
+
|
|
23
|
+
This version is straightforward, highlights the key issues with the official Ladybug package, and explains the advantages of **ladybug-lite** in a way that's easy for users to understand. Feel free to tweak it as needed!
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import multiprocessing as mp
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import real_ladybug as lbug
|
|
4
|
+
import pickle
|
|
5
|
+
import traceback
|
|
6
|
+
|
|
7
|
+
def lbug_worker(db_path, conn_pipe):
|
|
8
|
+
try:
|
|
9
|
+
db = lbug.Database(db_path)
|
|
10
|
+
conn = lbug.Connection(db)
|
|
11
|
+
|
|
12
|
+
while True:
|
|
13
|
+
msg = conn_pipe.recv()
|
|
14
|
+
if msg == "__close__":
|
|
15
|
+
break
|
|
16
|
+
query = msg
|
|
17
|
+
try:
|
|
18
|
+
result = conn.execute(query)
|
|
19
|
+
df = result.get_as_df()
|
|
20
|
+
conn_pipe.send(("ok", pickle.dumps(df)))
|
|
21
|
+
result.close()
|
|
22
|
+
conn.close()
|
|
23
|
+
db.close()
|
|
24
|
+
except Exception as e:
|
|
25
|
+
conn_pipe.send(("error", traceback.format_exc()))
|
|
26
|
+
except Exception as e:
|
|
27
|
+
conn_pipe.send(("error", traceback.format_exc()))
|
|
28
|
+
finally:
|
|
29
|
+
conn_pipe.close()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class SafeLadybugSubprocess:
|
|
33
|
+
def __init__(self, db_path):
|
|
34
|
+
self.db_path = db_path
|
|
35
|
+
self.parent_conn, self.child_conn = mp.Pipe()
|
|
36
|
+
self.process = mp.Process(target=lbug_worker, args=(self.db_path, self.child_conn))
|
|
37
|
+
self.process.start()
|
|
38
|
+
|
|
39
|
+
def execute(self, query: str) -> pd.DataFrame:
|
|
40
|
+
self.parent_conn.send(query)
|
|
41
|
+
status, payload = self.parent_conn.recv()
|
|
42
|
+
if status == "ok":
|
|
43
|
+
return pickle.loads(payload)
|
|
44
|
+
else:
|
|
45
|
+
raise RuntimeError(f"Error in Ladybug subprocess:\n{payload}")
|
|
46
|
+
|
|
47
|
+
def close(self):
|
|
48
|
+
if self.process.is_alive():
|
|
49
|
+
self.parent_conn.send("__close__")
|
|
50
|
+
self.process.join()
|
|
51
|
+
self.parent_conn.close()
|
|
52
|
+
self.child_conn.close()
|
|
53
|
+
|
|
54
|
+
def __enter__(self):
|
|
55
|
+
return self
|
|
56
|
+
|
|
57
|
+
def __exit__(self, *args):
|
|
58
|
+
self.close()
|