@quenty/datastore 7.23.0 → 7.23.1-canary.402.5852ffd.0

package/CHANGELOG.md

@@ -3,6 +3,26 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [7.23.1-canary.402.5852ffd.0](https://github.com/Quenty/NevermoreEngine/compare/@quenty/datastore@7.23.0...@quenty/datastore@7.23.1-canary.402.5852ffd.0) (2023-08-16)
+
+
+### Bug Fixes
+
+* Fix additional components but data store writing ping-pings back and forth ([16d024d](https://github.com/Quenty/NevermoreEngine/commit/16d024d226057f75249e326be6941db2d519dae7))
+* More data store improvements ([42ab23d](https://github.com/Quenty/NevermoreEngine/commit/42ab23d66a40aa81d034974e57234ee5337a208d))
+
+
+### Features
+
+* DataStores appear to be working, but require more testing ([d93a6b9](https://github.com/Quenty/NevermoreEngine/commit/d93a6b96da1c3a1ad411cedfcf6c9df571fd9a1f))
+* More untested datastore syncing code ([8dd4cbe](https://github.com/Quenty/NevermoreEngine/commit/8dd4cbe00500933ef6d9fc98ed34c94a5eb34943))
+* Semi-broken datastore changes ([351c44d](https://github.com/Quenty/NevermoreEngine/commit/351c44d48d1f7d7e26b8fbb30a14c7bb4cbac0f1))
+* Unfinished datastore changes ([0bbd600](https://github.com/Quenty/NevermoreEngine/commit/0bbd6005148787f0d06e22650f67e9886318f2f8))
+
+
+
+
+
 # [7.23.0](https://github.com/Quenty/NevermoreEngine/compare/@quenty/datastore@7.22.0...@quenty/datastore@7.23.0) (2023-08-01)
 
 **Note:** Version bump only for package @quenty/datastore

package/README.md

@@ -15,6 +15,17 @@ This system is a reliable datastore system designed with promises and asyncronio
 
 <div align="center"><a href="https://quenty.github.io/NevermoreEngine/api/DataStore">View docs →</a></div>
 
+## Executive overiew
+This datastore prevents data loss by being explicit about what we're writing to, and only modifying the data that exists there instead of modifying the whole structure.
+
+## How syncing works
+Sometimes datastores (like a global game data store) need to be synced live instead of upon server or player start. This is if we expect multiple servers to write to the same datastore at once we can use thie sync method to 
+
+Syncing is like saving. However, instead of treating the current datastore as a session lock, we load in additional data from our "source-of-truth". From here, we merge that data into the datastore, which means both clearing any matching write tokens that our sync says is done. 
+
+This is best for a "shared" memory that can be temporarily not correct. Deleting with a sync is less effective.
+
+
 ## Installation
 ```
 npm install @quenty/datastore --save

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@quenty/datastore",
-  "version": "7.23.0",
+  "version": "7.23.1-canary.402.5852ffd.0",
   "description": "Quenty's Datastore implementation for Roblox",
   "keywords": [
     "Roblox",
@@ -26,18 +26,21 @@
     "Quenty"
   ],
   "dependencies": {
-    "@quenty/baseobject": "^6.2.1",
-    "@quenty/bindtocloseservice": "^2.18.0",
-    "@quenty/loader": "^6.2.1",
-    "@quenty/maid": "^2.5.0",
-    "@quenty/promise": "^6.7.0",
-    "@quenty/rx": "^7.14.0",
-    "@quenty/signal": "^2.4.0",
-    "@quenty/symbol": "^2.2.0",
-    "@quenty/table": "^3.2.0"
+    "@quenty/baseobject": "6.2.2-canary.402.5852ffd.0",
+    "@quenty/bindtocloseservice": "2.18.1-canary.402.5852ffd.0",
+    "@quenty/loader": "6.2.2-canary.402.5852ffd.0",
+    "@quenty/maid": "2.5.0",
+    "@quenty/math": "2.4.1-canary.402.5852ffd.0",
+    "@quenty/promise": "6.7.1-canary.402.5852ffd.0",
+    "@quenty/rx": "7.14.1-canary.402.5852ffd.0",
+    "@quenty/servicebag": "6.8.1-canary.402.5852ffd.0",
+    "@quenty/signal": "2.4.0",
+    "@quenty/symbol": "2.2.0",
+    "@quenty/table": "3.2.1-canary.402.5852ffd.0",
+    "@quenty/valueobject": "7.21.1-canary.402.5852ffd.0"
   },
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "839b4d929e4d6154aadf719f1ecfb9add4c8b247"
+  "gitHead": "5852ffdd4ee4f857a5b7086330d59d852de49b86"
 }

package/src/Server/DataStore.lua

@@ -71,13 +71,14 @@ local DataStoreStage = require("DataStoreStage")
 local Maid = require("Maid")
 local Promise = require("Promise")
 local Signal = require("Signal")
+local Math = require("Math")
+local ValueObject = require("ValueObject")
+local Rx = require("Rx")
 
-local DEBUG_WRITING = false
+local DEFAULT_DEBUG_WRITING = false
 
-local AUTO_SAVE_TIME = 60*5
-local CHECK_DIVISION = 15
-local JITTER = 20 -- Randomly assign jitter so if a ton of players join at once we don't hit the datastore at once
-local DEFAULT_CACHE_TIME_SECONDS = math.huge
+local DEFAULT_AUTO_SAVE_TIME_SECONDS = 60*5
+local DEFAULT_JITTER_PROPORTION = 0.1 -- Randomly assign jitter so if a ton of players join at once we don't hit the datastore at once
 
 local DataStore = setmetatable({}, DataStoreStage)
 DataStore.ClassName = "DataStore"
@@ -85,16 +86,35 @@ DataStore.__index = DataStore
 
 --[=[
 	Constructs a new DataStore. See [DataStoreStage] for more API.
+
+	```lua
+	local dataStore = serviceBag:GetService(PlayerDataStoreService):PromiseDataStore(player):Yield()
+	```
+
 	@param robloxDataStore DataStore
 	@param key string
 	@return DataStore
 ]=]
 function DataStore.new(robloxDataStore, key)
-	local self = setmetatable(DataStoreStage.new(), DataStore)
+	local self = setmetatable(DataStoreStage.new(key), DataStore)
 
 	self._key = key or error("No key")
 	self._robloxDataStore = robloxDataStore or error("No robloxDataStore")
-	self._cacheTimeSeconds = DEFAULT_CACHE_TIME_SECONDS
+	self._debugWriting = DEFAULT_DEBUG_WRITING
+
+	self._autoSaveTimeSeconds = ValueObject.new(DEFAULT_AUTO_SAVE_TIME_SECONDS)
+	self._maid:GiveTask(self._autoSaveTimeSeconds)
+
+	self._jitterProportion = ValueObject.new(DEFAULT_JITTER_PROPORTION, "number")
+	self._maid:GiveTask(self._jitterProportion)
+
+	self._syncOnSave = ValueObject.new(false, "boolean")
+	self._maid:GiveTask(self._syncOnSave)
+
+	self._loadedOk = ValueObject.new(false, "boolean")
+	self._maid:GiveTask(self._loadedOk)
+
+	self._userIdList = nil
 
 	if self._key == "" then
 		error("[DataStore] - Key cannot be an empty string")
@@ -108,41 +128,20 @@ function DataStore.new(robloxDataStore, key)
 	self.Saving = Signal.new() -- :Fire(promise)
 	self._maid:GiveTask(self.Saving)
 
-	task.spawn(function()
-		while self.Destroy do
-			for _=1, CHECK_DIVISION do
-				task.wait(AUTO_SAVE_TIME/CHECK_DIVISION)
-				if not self.Destroy then
-					break
-				end
-			end
-
-			if not self.Destroy then
-				break
-			end
-
-			-- Apply additional jitter on auto-save
-			task.wait(math.random(1, JITTER))
-
-			if not self.Destroy then
-				break
-			end
-
-			self:Save()
-		end
-	end)
+	self:_setupAutoSaving()
 
 	return self
 end
 
 --[=[
-	Sets how long the datastore will cache for
-	@param cacheTimeSeconds number?
+	Set to true to debug writing this data store
+
+	@param debugWriting boolean
 ]=]
-function DataStore:SetCacheTime(cacheTimeSeconds)
-	assert(type(cacheTimeSeconds) == "number" or cacheTimeSeconds == nil, "Bad cacheTimeSeconds")
+function DataStore:SetDoDebugWriting(debugWriting)
+	assert(type(debugWriting) == "boolean", "Bad debugWriting")
 
-	self._cacheTimeSeconds = cacheTimeSeconds or DEFAULT_CACHE_TIME_SECONDS
+	self._debugWriting = debugWriting
 end
 
 --[=[
@@ -153,16 +152,39 @@ function DataStore:GetFullPath()
 	return ("RobloxDataStore@%s"):format(self._key)
 end
 
+--[=[
+	How frequent the data store will autosave (or sync) to the cloud. If set to nil then the datastore
+	will not do any syncing.
+
+	@param autoSaveTimeSeconds number | nil
+]=]
+function DataStore:SetAutoSaveTimeSeconds(autoSaveTimeSeconds)
+	assert(type(autoSaveTimeSeconds) == "number" or autoSaveTimeSeconds == nil, "Bad autoSaveTimeSeconds")
+
+	self._autoSaveTimeSeconds.Value = autoSaveTimeSeconds
+end
+
+--[=[
+	How frequent the data store will autosave (or sync) to the cloud
+
+	@param syncEnabled boolean
+]=]
+function DataStore:SetSyncOnSave(syncEnabled)
+	assert(type(syncEnabled) == "boolean", "Bad syncEnabled")
+
+	self._syncOnSave.Value = syncEnabled
+end
+
 --[=[
 	Returns whether the datastore failed.
 	@return boolean
 ]=]
 function DataStore:DidLoadFail()
-	if not self._loadPromise then
+	if not self._firstLoadPromise then
 		return false
 	end
 
-	if self._loadPromise:IsRejected() then
+	if self._firstLoadPromise:IsRejected() then
 		return true
 	end
 
@@ -175,7 +197,7 @@ end
 	@return Promise<boolean>
 ]=]
 function DataStore:PromiseLoadSuccessful()
-	return self._maid:GivePromise(self:_promiseLoad()):Then(function()
+	return self._maid:GivePromise(self:PromiseViewUpToDate()):Then(function()
 		return true
 	end, function()
 		return false
@@ -187,67 +209,208 @@ end
 	@return Promise
 ]=]
 function DataStore:Save()
-	if self:DidLoadFail() then
-		warn("[DataStore] - Not saving, failed to load")
-		return Promise.rejected("Load not successful, not saving")
+	return self:_syncData(false)
+end
+
+--[=[
+	Same as saving the data but it also loads fresh data from the datastore, which may consume
+	additional data-store query calls.
+
+	@return Promise
+]=]
+function DataStore:Sync()
+	return self:_syncData(true)
+end
+
+--[=[
+	Sets the user id list associated with this datastore. Can be useful for GDPR compliance.
+
+	@param userIdList { number } | nil
+]=]
+function DataStore:SetUserIdList(userIdList)
+	assert(type(userIdList) == "table" or userIdList == nil, "Bad userIdList")
+
+	self._userIdList = userIdList
+end
+
+--[=[
+	Returns a list of user ids or nil
+
+	@return { number } | nil
+]=]
+function DataStore:GetUserIdList()
+	return self._userIdList
+end
+
+--[=[
+	Overridden helper method for data store stage below.
+
+	@return Promise
+]=]
+function DataStore:PromiseViewUpToDate()
+	if self._firstLoadPromise then
+		return self._firstLoadPromise
 	end
 
-	if DEBUG_WRITING then
-		print("[DataStore.Save] - Starting save routine")
+	self._firstLoadPromise = self:_promiseGetAsyncNoCache()
+
+	self._firstLoadPromise:Tap(function()
+		self._loadedOk.Value = true
+	end)
+
+	return self._firstLoadPromise
+end
+
+function DataStore:_setupAutoSaving()
+	local startTime = os.clock()
+
+	self._maid:GiveTask(Rx.combineLatest({
+		autoSaveTimeSeconds = self._autoSaveTimeSeconds:Observe();
+		jitterProportion = self._jitterProportion:Observe();
+		syncOnSave = self._syncOnSave:Observe();
+		loadedOk = self._loadedOk:Observe();
+	}):Subscribe(function(state)
+		if state.autoSaveTimeSeconds and state.loadedOk then
+			local maid = Maid.new()
+			if self._debugWriting then
+				print("Auto-saving loop started")
+			end
+
+			-- TODO: First jitter is way noisier to differentiate servers
+			maid:GiveTask(task.spawn(function()
+				while true do
+					local jitterBase = math.random()
+					local timeElapsed = os.clock() - startTime
+					local totalWaitTime = Math.jitter(state.autoSaveTimeSeconds, state.jitterProportion*state.autoSaveTimeSeconds, jitterBase)
+					local timeRemaining = totalWaitTime - timeElapsed
+
+					if timeRemaining > 0 then
+						task.wait(timeRemaining)
+					end
+
+					startTime = os.clock()
+
+					if state.syncOnSave then
+						self:Sync()
+					else
+						self:Save()
+					end
+
+					task.wait(0.1)
+				end
+			end))
+
+			self._maid._autoSavingMaid = maid
+		else
+			self._maid._autoSavingMaid = nil
+		end
+	end))
+end
+
+function DataStore:_syncData(doMergeNewData)
+	if self:DidLoadFail() then
+		warn("[DataStore] - Not syncing, failed to load")
+		return Promise.rejected("Load not successful, not syncing")
 	end
 
-	-- Avoid constructing promises for every callback down the datastore
-	-- upon save.
-	return (self:_promiseInvokeSavingCallbacks() or Promise.resolved())
+	return self._maid:GivePromise(self:PromiseViewUpToDate())
+		:Then(function()
+			return self._maid:GivePromise(self:PromiseInvokeSavingCallbacks())
+		end)
 		:Then(function()
 			if not self:HasWritableData() then
+				if doMergeNewData then
+					-- Reads are cheaper than update async calls
+					return self:_promiseGetAsyncNoCache()
+				end
+
 				-- Nothing to save, don't update anything
-				if DEBUG_WRITING then
-					print("[DataStore.Save] - Not saving, nothing staged")
+				if self._debugWriting then
+					print("[DataStore] - Not saving, nothing staged")
 				end
+
 				return nil
 			else
-				return self:_saveData(self:GetNewWriter())
+				return self:_doDataSync(self:GetNewWriter(), doMergeNewData)
 			end
 		end)
 end
 
---[=[
-	Loads data. This returns the originally loaded data.
-	@param keyName string
-	@param defaultValue any?
-	@return any?
-]=]
-function DataStore:Load(keyName, defaultValue)
-	return self:_promiseLoad()
-		:Then(function(data)
-			return self:_afterLoadGetAndApplyStagedData(keyName, data, defaultValue)
-		end)
-end
+function DataStore:_doDataSync(writer, doMergeNewData)
+	assert(type(doMergeNewData) == "boolean", "Bad doMergeNewData")
+
+	-- Cache user id list
+	writer:SetUserIdList(self:GetUserIdList())
 
-function DataStore:_saveData(writer)
 	local maid = Maid.new()
 
 	local promise = Promise.new()
-	promise:Resolve(maid:GivePromise(DataStorePromises.updateAsync(self._robloxDataStore, self._key, function(data)
-		if promise:IsRejected() then
-			-- Cancel if we have another request
-			return nil
+
+	if writer:IsCompleteWipe() then
+		if self._debugWriting then
+			print(string.format("[DataStore] - DataStorePromises.removeAsync(%q)", self._key))
 		end
 
-		data = writer:WriteMerge(data or {})
-		assert(data ~= DataStoreDeleteToken, "Cannot delete from UpdateAsync")
+		-- This is, of course, dangerous, because we won't merge
+		promise:Resolve(maid:GivePromise(DataStorePromises.removeAsync(self._robloxDataStore, self._key)):Then(function()
+			if doMergeNewData then
+				-- Write our data
+				self:MarkDataAsSaved(writer)
 
-		if DEBUG_WRITING then
-			print("[DataStore] - Writing", game:GetService("HttpService"):JSONEncode(data))
+				-- Do syncing after
+				return self:_promiseGetAsyncNoCache()
+			end
+		end))
+	else
+		if self._debugWriting then
+			print(string.format("[DataStore] - DataStorePromises.updateAsync(%q)", self._key))
 		end
 
-		return data
-	end, function(err)
+		promise:Resolve(maid:GivePromise(DataStorePromises.updateAsync(self._robloxDataStore, self._key, function(original, datastoreKeyInfo)
+			if promise:IsRejected() then
+				-- Cancel if we have another request
+				return nil
+			end
+
+			local diffSnapshot
+			if doMergeNewData then
+				diffSnapshot = writer:ComputeDiffSnapshot(original)
+			end
+
+			local result = writer:WriteMerge(original)
+
+			if result == DataStoreDeleteToken or result == nil then
+				result = {}
+			end
+
+			if self._debugWriting then
+				print("[DataStore] - Writing", result)
+			end
+
+			if doMergeNewData then
+				-- This prevents resaving at high frequency
+				self:MarkDataAsSaved(writer)
+				self:MergeDiffSnapshot(diffSnapshot)
+			end
+
+			local userIdList = writer:GetUserIdList()
+			if datastoreKeyInfo then
+				userIdList = datastoreKeyInfo:GetUserIds()
+			end
+
+			local metadata = nil
+			if datastoreKeyInfo then
+				metadata = datastoreKeyInfo:GetMetadata()
+			end
+
+			return result, userIdList, metadata
+		end)))
+	end
+
+	promise:Tap(nil, function(err)
 		-- Might be caused by Maid rejecting state
-		warn("[DataStore] - Failed to UpdateAsync data", err)
-		return Promise.rejected(err)
-	end)))
+		warn("[DataStore] - Failed to sync data", err)
+	end)
 
 	self._maid._saveMaid = maid
 
@@ -258,27 +421,23 @@ function DataStore:_saveData(writer)
 	return promise
 end
 
-function DataStore:_promiseLoad()
-	if self._loadPromise then
-		return self._loadPromise
-	end
-
-	self._loadPromise = self._maid:GivePromise(DataStorePromises.getAsync(self._robloxDataStore, self._key)
-		:Then(function(data)
-			if data == nil then
-				return {}
-			elseif type(data) == "table" then
-				return data
-			else
-				return Promise.rejected("Failed to load data. Wrong type '" .. type(data) .. "'")
-			end
-		end, function(err)
-			-- Log:
-			warn("[DataStore] - Failed to GetAsync data", err)
+function DataStore:_promiseGetAsyncNoCache()
+	return self._maid:GivePromise(DataStorePromises.getAsync(self._robloxDataStore, self._key))
+		:Catch(function(err)
+			warn(string.format("DataStorePromises.getAsync(%q) -> warning - ", self._key), err)
 			return Promise.rejected(err)
-		end))
+		end)
+		:Then(function(data)
+			local writer = self:GetNewWriter()
+			local diffSnapshot = writer:ComputeDiffSnapshot(data)
 
-	return self._loadPromise
+			self:MergeDiffSnapshot(diffSnapshot)
+
+			if self._debugWriting then
+				print(string.format("DataStorePromises.getAsync(%q) -> Got ", self._key), data, "with diff snapshot", diffSnapshot, "to view", self._viewSnapshot)
+				-- print(string.format("DataStorePromises.getAsync(%q) -> Got ", self._key), data)
+			end
+		end)
 end
 
 return DataStore

package/src/Server/GameDataStoreService.lua

@@ -10,6 +10,7 @@ local require = require(script.Parent.loader).load(script)
 local DataStore = require("DataStore")
 local DataStorePromises = require("DataStorePromises")
 local Maid = require("Maid")
+local Promise = require("Promise")
 
 local GameDataStoreService = {}
 GameDataStoreService.ServiceName = "GameDataStoreService"
@@ -30,11 +31,16 @@ function GameDataStoreService:PromiseDataStore()
 
 	self._dataStorePromise = self:_promiseRobloxDataStore()
 		:Then(function(robloxDataStore)
+			-- Live sync this stuff pretty frequently
 			local dataStore = DataStore.new(robloxDataStore, self:_getKey())
+			dataStore:SetSyncOnSave(true)
+			dataStore:SetAutoSaveTimeSeconds(15)
 			self._maid:GiveTask(dataStore)
 
 			self._maid:GiveTask(self._bindToCloseService:RegisterPromiseOnCloseCallback(function()
-				return dataStore:Save()
+				return Promise.defer(function(resolve)
+					return resolve(dataStore:Save())
+				end)
 			end))
 
 			return dataStore

package/src/Server/Modules/DataStoreSnapshotUtils.lua

@@ -0,0 +1,13 @@
+--[=[
+	@class DataStoreSnapshotUtils
+]=]
+
+local require = require(script.Parent.loader).load(script)
+
+local DataStoreSnapshotUtils = {}
+
+function DataStoreSnapshotUtils.isEmptySnapshot(snapshot)
+	return type(snapshot) == "table" and next(snapshot) == nil
+end
+
+return DataStoreSnapshotUtils