@stacksjs/ts-cloud 0.7.93 → 0.7.95

package/dist/ui-src/pages/server/database.stx

@@ -8,6 +8,100 @@ import { requestJson } from '../../functions/requestJson.ts'
 
 // Reactive database management - create databases/users, live lists.
 const engine = state('mysql')
+
+// Engines that only exist as an external cluster. There is no on-box engine
+// to administer, so the create/backup forms below are hidden rather than
+// left to fail: they would shell into a pantry socket that cannot exist.
+const EXTERNAL_ENGINES = ['singlestore', 'vitess']
+const isExternal = derived(() => EXTERNAL_ENGINES.includes(engine()))
+const isVitess = derived(() => engine() === 'vitess')
+
+// Vitess topology, read from vtgate. Read-only: resharding is a vtctld
+// operation with real blast radius and does not belong behind a button.
+const keyspaces = state([])
+const shards = state([])
+const tablets = state([])
+const unhealthy = state([])
+const missingPrimary = state([])
+const vitessError = state('')
+async function loadVitess() {
+  if (!isVitess()) return
+  try {
+    const res = await fetch('/api/databases/vitess')
+    const d = await res.json()
+    if (d.ok) {
+      keyspaces.set(d.keyspaces || [])
+      shards.set(d.shards || [])
+      tablets.set(d.tablets || [])
+      unhealthy.set(d.unhealthy || [])
+      missingPrimary.set(d.missingPrimary || [])
+      vitessError.set('')
+    } else {
+      vitessError.set(d.error || 'Could not reach vtgate.')
+    }
+  } catch (e) { vitessError.set((e && e.message) || String(e)) }
+}
+
+// Online-DDL migrations. Read through vtgate, so this works even on a
+// cluster with no vtctld address configured.
+const migrations = state([])
+const failedMigs = state([])
+const runningMigs = state([])
+async function loadMigrations() {
+  if (!isVitess()) return
+  try {
+    const res = await fetch('/api/databases/vitess/migrations')
+    const d = await res.json()
+    if (d.ok) { migrations.set(d.migrations || []); failedMigs.set(d.failed || []); runningMigs.set(d.running || []) }
+  } catch {}
+}
+async function migrationAction(uuid, action) {
+  if (busy()) return
+  busy.set(true)
+  show(action + ' ' + uuid + '...')
+  try {
+    const b = await requestJson('/api/databases/vitess/migrations/action', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ uuid, action }) })
+    show((b.ok ? 'OK ' + action : 'FAILED') + '\n\n' + (b.stdout || '') + (b.stderr ? '\n' + b.stderr : '') + (b.error ? '\n' + b.error : ''))
+    if (b.ok) loadMigrations()
+  } catch (e) { show('FAILED\n\n' + ((e && e.message) || e)) }
+  finally { busy.set(false) }
+}
+
+// Topology operations. These need vtctld, so each reports clearly when no
+// control-plane address is configured rather than looking like a glitch.
+const ksName = state('')
+const ksSharded = state(true)
+const vschemaKeyspace = state('')
+const vschemaJson = state('')
+const ddlSql = state('')
+async function post(path, payload, label) {
+  if (busy()) return
+  busy.set(true)
+  show(label + '...')
+  try {
+    const b = await requestJson(path, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(payload) })
+    show((b.ok ? 'OK ' + label : 'FAILED') + '\n\n' + (b.stdout || '') + (b.stderr ? '\n' + b.stderr : '') + (b.error ? '\n' + b.error : ''))
+    return b.ok
+  } catch (e) { show('FAILED\n\n' + ((e && e.message) || e)); return false }
+  finally { busy.set(false) }
+}
+async function createKeyspace() {
+  if (await post('/api/databases/vitess/keyspaces', { name: ksName(), sharded: ksSharded() }, 'create keyspace ' + ksName())) {
+    ksName.set(''); load(); loadVitess()
+  }
+}
+async function applyVSchema() {
+  if (await post('/api/databases/vitess/vschema', { keyspace: vschemaKeyspace(), vschema: vschemaJson() }, 'apply VSchema to ' + vschemaKeyspace()))
+    loadVitess()
+}
+async function applySchema() {
+  if (await post('/api/databases/vitess/schema', { sql: ddlSql() }, 'apply schema change')) {
+    ddlSql.set(''); loadMigrations()
+  }
+}
+async function installClient() {
+  await post('/api/databases/vitess/install-client', {}, 'install vtctldclient')
+}
 const databases = state([])
 const users = state([])
 const dbError = state('')
@@ -70,7 +164,7 @@ async function backupDb(name) {
   } catch (e) { show('FAILED\n\n' + ((e && e.message) || e)) }
   finally { busy.set(false) }
 }
-onMount(() => { load(); loadBackups() })
+onMount(async () => { await load(); loadBackups(); loadVitess(); loadMigrations() })
 </script>
 <!DOCTYPE html>
 <html lang="en" data-theme="dark">
@@ -80,10 +174,24 @@ onMount(() => { load(); loadBackups() })
   <div class="wrap">
     <div class="crumbs"><a href="/">Server</a> <span class="sep">/</span> Databases</div>
     <header class="page">
-      <div><h1>Databases &amp; users</h1><p>Create databases and app users on the managed engine (<span class="mono">{{ engine() }}</span>).</p></div>
+      <div>
+        <h1>Databases &amp; users</h1>
+        <p @show="!isExternal()">Create databases and app users on the managed engine (<span class="mono">{{ engine() }}</span>).</p>
+        <p @show="isExternal()"><span class="mono">{{ engine() }}</span> is an external cluster. This box does not host the engine, so databases and users are managed on the cluster itself.</p>
+      </div>
     </header>
 
-    <div class="cols section">
+    <div class="section" @show="isVitess()">
+      <div class="panel">
+        <div class="kv"><span>Engine</span> <b class="mono">Vitess</b></div>
+        <div class="kv"><span>Endpoint</span> <b class="mono">vtgate</b></div>
+        <div class="kv"><span>Keyspaces</span> <b class="mono">{{ keyspaces().length }}</b></div>
+        <div class="kv"><span>Shards</span> <b class="mono">{{ shards().length }}</b></div>
+        <div class="kv"><span>Tablets</span> <b class="mono">{{ tablets().length }}</b></div>
+      </div>
+    </div>
+
+    <div class="cols section" @show="!isExternal()">
       <div>
         <h2>Create database</h2>
         <div class="panel">
@@ -121,16 +229,18 @@ onMount(() => { load(); loadBackups() })
 
     <div class="cols section">
       <div>
-        <h2>Databases</h2>
+        <h2 @show="!isVitess()">Databases</h2>
+        <h2 @show="isVitess()">Keyspaces</h2>
         <div class="panel">
           <template :for="n in databases()">
-            <div class="kv"><span class="mono">{{ n }}</span> <button class="btn ghost sm" type="button" :disabled="busy()" @click="backupDb(n)" :aria-label="'Back up ' + n">Back up</button></div>
+            <div class="kv"><span class="mono">{{ n }}</span> <button class="btn ghost sm" type="button" @show="!isExternal()" :disabled="busy()" @click="backupDb(n)" :aria-label="'Back up ' + n">Back up</button></div>
           </template>
           <span class="item" @show="databases().length === 0 && dbError() !== ''">{{ dbError() }}</span>
-          <span class="item" @show="databases().length === 0 && dbError() === ''">no databases</span>
+          <span class="item" @show="databases().length === 0 && dbError() === '' && !isVitess()">no databases</span>
+          <span class="item" @show="databases().length === 0 && dbError() === '' && isVitess()">no keyspaces</span>
         </div>
       </div>
-      <div>
+      <div @show="!isExternal()">
         <h2>Users</h2>
         <div class="panel">
           <div class="svc">
@@ -140,8 +250,145 @@ onMount(() => { load(); loadBackups() })
           </div>
         </div>
       </div>
+      <div @show="isVitess()">
+        <h2>Shards</h2>
+        <div class="panel">
+          <template :for="s in shards()">
+            <div class="kv"><span class="mono">{{ s.keyspace }}/{{ s.shard }}</span></div>
+          </template>
+          <span class="item" @show="shards().length === 0">no shards reported</span>
+        </div>
+      </div>
+    </div>
+
+    <div class="section" @show="isVitess()">
+      <h2>Tablets</h2>
+
+      <div class="panel" @show="missingPrimary().length > 0">
+        <div class="compact empty">
+          <strong>{{ missingPrimary().length }} shard(s) have no serving PRIMARY</strong>
+          <span>Vitess routes writes to a shard's primary. Without one the shard still answers reads and silently fails writes. Check tablet health, then reparent with <span class="mono">vtctldclient PlannedReparentShard</span>.</span>
+        </div>
+        <template :for="s in missingPrimary()">
+          <div class="kv"><span class="mono">{{ s.keyspace }}/{{ s.shard }}</span> <b class="mono">no primary</b></div>
+        </template>
+      </div>
+
+      <div class="panel" @show="unhealthy().length > 0">
+        <div class="compact empty"><strong>{{ unhealthy().length }} tablet(s) not serving</strong></div>
+        <template :for="t in unhealthy()">
+          <div class="kv"><span class="mono">{{ t.keyspace }}/{{ t.shard }} {{ t.alias }}</span> <b class="mono">{{ t.type }} {{ t.state }}</b></div>
+        </template>
+      </div>
+
+      <div class="panel">
+        <template :for="t in tablets()">
+          <div class="kv"><span class="mono">{{ t.keyspace }}/{{ t.shard }} {{ t.alias }}</span> <b class="mono" style="font-size:11px;color:var(--txt3)">{{ t.type }} {{ t.state }} {{ t.hostname }}</b></div>
+        </template>
+        <div class="compact empty" @show="tablets().length === 0 && vitessError() === ''"><strong>No tablets reported</strong><span>vtgate returned no tablets for this cluster.</span></div>
+        <div class="compact empty" @show="vitessError() !== ''"><strong>Could not read the Vitess topology</strong><span>{{ vitessError() }}</span></div>
+      </div>
+    </div>
+
+    <div class="section" @show="isVitess()">
+      <h2>Schema migrations</h2>
+
+      <div class="panel" @show="failedMigs().length > 0">
+        <div class="compact empty"><strong>{{ failedMigs().length }} migration(s) failed</strong><span>A failed online DDL leaves the table unchanged. Retry once the cause is fixed, or cancel to discard it.</span></div>
+        <template :for="m in failedMigs()">
+          <div class="kv">
+            <span class="mono">{{ m.keyspace }}/{{ m.shard }} {{ m.table }}</span>
+            <span>
+              <button class="btn ghost sm" type="button" :disabled="busy()" @click="migrationAction(m.uuid, 'retry')">Retry</button>
+              <button class="btn ghost sm" type="button" :disabled="busy()" @click="migrationAction(m.uuid, 'cleanup')">Clean up</button>
+            </span>
+          </div>
+        </template>
+      </div>
+
+      <div class="panel" @show="runningMigs().length > 0">
+        <div class="compact empty"><strong>{{ runningMigs().length }} migration(s) in flight</strong></div>
+        <template :for="m in runningMigs()">
+          <div class="kv">
+            <span class="mono">{{ m.keyspace }}/{{ m.shard }} {{ m.table }} {{ m.progress }}</span>
+            <span>
+              <button class="btn ghost sm" type="button" :disabled="busy()" @click="migrationAction(m.uuid, 'complete')">Complete</button>
+              <button class="btn ghost sm" type="button" :disabled="busy()" @click="migrationAction(m.uuid, 'cancel')">Cancel</button>
+            </span>
+          </div>
+        </template>
+      </div>
+
+      <div class="panel">
+        <template :for="m in migrations()">
+          <div class="kv"><span class="mono">{{ m.table }} <b style="color:var(--txt3)">{{ m.keyspace }}/{{ m.shard }}</b></span> <b class="mono" style="font-size:11px;color:var(--txt3)">{{ m.status }} {{ m.progress }} {{ m.added }}</b></div>
+        </template>
+        <div class="compact empty" @show="migrations().length === 0"><strong>No schema migrations</strong><span>Changes applied below appear here with live progress.</span></div>
+      </div>
+
+      <h2>Apply a schema change</h2>
+      <div class="panel">
+        <form class="form-grid" @submit.prevent="applySchema()">
+          <div class="field-block wide"><label for="ddl-sql">DDL</label><input id="ddl-sql" placeholder="ALTER TABLE users ADD COLUMN nickname VARCHAR(64)" autocomplete="off" :value="ddlSql()" @input="ddlSql.set($event.target.value)"></div>
+          <div class="form-actions"><button class="btn" type="submit" :disabled="busy()">Apply online</button></div>
+        </form>
+        <p class="note">Runs as Vitess online DDL: applied shard by shard without locking the keyspace, revertible, and tracked above. A direct (locking) change is a config choice, not a button, because on a sharded keyspace it is an outage rather than a migration.</p>
+      </div>
+    </div>
+
+    <div class="section" @show="isVitess()">
+      <h2>Topology</h2>
+      <div class="cols">
+        <div>
+          <h3>Create keyspace</h3>
+          <div class="panel">
+            <form class="form-grid" @submit.prevent="createKeyspace()">
+              <div class="field-block wide"><label for="ks-name">Name</label><input id="ks-name" placeholder="commerce" autocomplete="off" :value="ksName()" @input="ksName.set($event.target.value)"></div>
+              <div class="field-block"><label for="ks-sharded">Sharded</label>
+                <select id="ks-sharded" :value="ksSharded() ? 'yes' : 'no'" @change="ksSharded.set($event.target.value === 'yes')">
+                  <option value="yes">sharded</option>
+                  <option value="no">unsharded</option>
+                </select>
+              </div>
+              <div class="form-actions"><button class="btn" type="submit" :disabled="busy()">Create keyspace</button></div>
+            </form>
+            <p class="note">A sharded keyspace is not routable until its VSchema names a vindex per table. Apply one next.</p>
+          </div>
+        </div>
+        <div>
+          <h3>Apply VSchema</h3>
+          <div class="panel">
+            <form class="form-grid" @submit.prevent="applyVSchema()">
+              <div class="field-block"><label for="vs-ks">Keyspace</label>
+                <select id="vs-ks" :value="vschemaKeyspace()" @change="vschemaKeyspace.set($event.target.value)">
+                  <option value="">(select)</option>
+                  <template :for="k in keyspaces()"><option :value="k">{{ k }}</option></template>
+                </select>
+              </div>
+              <div class="field-block wide"><label for="vs-json">VSchema JSON</label><input id="vs-json" placeholder='paste database/vschema.json' autocomplete="off" :value="vschemaJson()" @input="vschemaJson.set($event.target.value)"></div>
+              <div class="form-actions"><button class="btn" type="submit" :disabled="busy()">Apply VSchema</button></div>
+            </form>
+            <p class="note">Generate it with <span class="mono">buddy generate:vschema</span>, which derives the sharding topology from your model relationships.</p>
+          </div>
+        </div>
+      </div>
+      <div class="panel">
+        <div class="kv"><span>Control plane</span> <button class="btn ghost sm" type="button" :disabled="busy()" @click="installClient()">Install vtctldclient</button></div>
+        <p class="note">Creating a keyspace and applying a VSchema are vtctld operations, so they need <span class="mono">vitess.vtctldAddr</span> configured and the client installed. Everything else on this page works through vtgate alone.</p>
+      </div>
+    </div>
+
+    <div class="section" @show="isVitess()">
+      <h2>Setup</h2>
+      <div class="panel">
+        <div class="kv"><span>1. Point the app at vtgate</span> <b class="mono">DB_CONNECTION=vitess, DB_PORT=15306</b></div>
+        <div class="kv"><span>2. Generate the VSchema</span> <b class="mono">buddy generate:vschema</b></div>
+        <div class="kv"><span>3. Apply it</span> <b class="mono">vtctldclient ApplyVSchema --vschema-file database/vschema.json &lt;keyspace&gt;</b></div>
+        <div class="kv"><span>4. Apply schema changes</span> <b class="mono">vtctldclient ApplySchema --sql-file &lt;file&gt; &lt;keyspace&gt;</b></div>
+        <p class="note">Port 15306 is vtgate's. Connecting on 3306 reaches one tablet's MySQL and bypasses sharding, which looks like a working connection. A sharded keyspace also rejects foreign keys and AUTO_INCREMENT, so models need <span class="mono">useUuid</span>.</p>
+      </div>
     </div>
-    <div class="section">
+    <div class="section" @show="!isExternal()">
       <h2>Backups</h2>
       <div class="panel">
         <template :for="b in backups()">
@@ -151,7 +398,8 @@ onMount(() => { load(); loadBackups() })
       </div>
     </div>
 
-    <p class="note">Lists are read live from the engine via the app server. Create operations run as root over the pantry socket; apps connect with the credentials you set here. Per-database dumps are written to <span class="mono">/var/backups/ts-cloud/databases</span>.</p>
+    <p class="note" @show="!isExternal()">Lists are read live from the engine via the app server. Create operations run as root over the pantry socket; apps connect with the credentials you set here. Per-database dumps are written to <span class="mono">/var/backups/ts-cloud/databases</span>.</p>
+    <p class="note" @show="isExternal()">Read live from the cluster through the app server. Creating databases or users, and taking backups, are cluster-side operations and are not offered here - this box hosts no engine to administer.</p>
   </div>
 </body>
 </html>

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@stacksjs/ts-cloud",
   "type": "module",
-  "version": "0.7.93",
+  "version": "0.7.95",
   "description": "A lightweight, performant infrastructure-as-code library and CLI for deploying both server-based (EC2) and serverless applications.",
   "author": "Chris Breuer <chris@stacksjs.com>",
   "license": "MIT",
@@ -90,8 +90,8 @@
   },
   "dependencies": {
     "@stacksjs/ts-xml": "^0.1.0",
-    "@ts-cloud/aws-types": "0.7.93",
-    "@ts-cloud/core": "0.7.93"
+    "@ts-cloud/aws-types": "0.7.95",
+    "@ts-cloud/core": "0.7.95"
   },
   "devDependencies": {
     "better-dx": "^0.2.16",