@quenty/datastore 8.0.0-canary.367.e9fdcbc.0 → 8.0.0

package/src/Server/Modules/DataStoreWriter.lua

@@ -8,6 +8,11 @@ local require = require(script.Parent.loader).load(script)
 
 local Table = require("Table")
 local DataStoreDeleteToken = require("DataStoreDeleteToken")
+local Symbol = require("Symbol")
+local Set = require("Set")
+local DataStoreSnapshotUtils = require("DataStoreSnapshotUtils")
+
+local UNSET_TOKEN = Symbol.named("unsetValue")
 
 local DataStoreWriter = {}
 DataStoreWriter.ClassName = "DataStoreWriter"
@@ -16,12 +21,17 @@ DataStoreWriter.__index = DataStoreWriter
 --[=[
 	Constructs a new DataStoreWriter. In general, you will not use this API directly.
 
+	@param debugName string
 	@return DataStoreWriter
 ]=]
-function DataStoreWriter.new()
+function DataStoreWriter.new(debugName)
 	local self = setmetatable({}, DataStoreWriter)
 
-	self._rawSetData = {}
+	self._debugName = assert(debugName, "No debugName")
+	self._saveDataSnapshot = UNSET_TOKEN
+	self._fullBaseDataSnapshot = UNSET_TOKEN
+	self._userIdList = UNSET_TOKEN
+
 	self._writers = {}
 
 	return self
@@ -29,10 +39,40 @@ end
 
 --[=[
 	Sets the ray data to write
-	@param data table
+	@param saveDataSnapshot table | any
 ]=]
-function DataStoreWriter:SetRawData(data)
-	self._rawSetData = Table.deepCopy(data)
+function DataStoreWriter:SetSaveDataSnapshot(saveDataSnapshot)
+	assert(type(saveDataSnapshot) ~= "table" or table.isfrozen(saveDataSnapshot), "saveDataSnapshot should be frozen")
+
+	if saveDataSnapshot == DataStoreDeleteToken then
+		self._saveDataSnapshot = DataStoreDeleteToken
+	elseif type(saveDataSnapshot) == "table" then
+		self._saveDataSnapshot = Table.deepCopy(saveDataSnapshot)
+	else
+		self._saveDataSnapshot = saveDataSnapshot
+	end
+end
+
+function DataStoreWriter:GetDataToSave()
+	if self._saveDataSnapshot == UNSET_TOKEN then
+		return nil
+	end
+
+	return self._saveDataSnapshot
+end
+
+function DataStoreWriter:GetSubWritersMap()
+	return self._writers
+end
+
+function DataStoreWriter:SetFullBaseDataSnapshot(fullBaseDataSnapshot)
+	assert(type(fullBaseDataSnapshot) ~= "table" or table.isfrozen(fullBaseDataSnapshot), "fullBaseDataSnapshot should be frozen")
+
+	if fullBaseDataSnapshot == DataStoreDeleteToken then
+		error("[DataStoreWriter] - fullBaseDataSnapshot should not be a delete token")
+	end
+
+	self._fullBaseDataSnapshot = fullBaseDataSnapshot
 end
 
 --[=[
@@ -40,7 +80,7 @@ end
 	@param name string
 	@param writer DataStoreWriter
 ]=]
-function DataStoreWriter:AddWriter(name, writer)
+function DataStoreWriter:AddSubWriter(name, writer)
 	assert(type(name) == "string", "Bad name")
 	assert(not self._writers[name], "Writer already exists for name")
 	assert(writer, "Bad writer")
@@ -49,37 +89,207 @@ function DataStoreWriter:AddWriter(name, writer)
 end
 
 --[=[
-	Merges the new data into the original value
+	Gets a sub writer
 
-	@param original table?
-	@return table -- The original table
+	@param name string
+	@return DataStoreWriter
 ]=]
-function DataStoreWriter:WriteMerge(original)
-	original = original or {}
+function DataStoreWriter:GetWriter(name)
+	assert(type(name) == "string", "Bad name")
+
+	return self._writers[name]
+end
 
-	for key, value in pairs(self._rawSetData) do
-		if value == DataStoreDeleteToken then
-			original[key] = nil
+--[=[
+	Merges the incoming data.
+
+	Won't really perform a delete operation because we can't be sure if we were suppose to have reified this stuff or not.
+
+	@param incoming any
+]=]
+function DataStoreWriter:ComputeDiffSnapshot(incoming)
+	assert(incoming ~= DataStoreDeleteToken, "Incoming value should not be DataStoreDeleteToken")
+
+	if type(incoming) == "table" then
+		local keys = Set.union(Set.fromKeys(self._writers), Set.fromKeys(incoming))
+
+		local baseSnapshot
+		if type(self._fullBaseDataSnapshot) == "table" then
+			baseSnapshot = self._fullBaseDataSnapshot
+			Set.unionUpdate(keys, Set.fromKeys(self._fullBaseDataSnapshot))
 		else
-			original[key] = value
+			baseSnapshot = {}
 		end
-	end
 
-	for key, writer in pairs(self._writers) do
-		if self._rawSetData[key] ~= nil then
-			warn(("[DataStoreWriter.WriteMerge] - Overwritting key %q already saved as rawData with a writer")
-				:format(tostring(key)))
+		local diffSnapshot = {}
+		for key, _ in pairs(keys) do
+			if self._writers[key] then
+				diffSnapshot[key] = self._writers[key]:ComputeDiffSnapshot(incoming[key])
+			else
+				diffSnapshot[key] = self:_computeValueDiff(baseSnapshot[key], incoming[key])
+			end
 		end
 
-		local result = writer:WriteMerge(original[key])
-		if result == DataStoreDeleteToken then
-			original[key] = nil
+		if not DataStoreSnapshotUtils.isEmptySnapshot(diffSnapshot) then
+			return table.freeze(diffSnapshot)
+		else
+			if next(keys) then
+				return nil -- No delta
+			else
+				return DataStoreDeleteToken
+			end
+		end
+	else
+		return self:_computeValueDiff(self._fullBaseDataSnapshot, incoming)
+	end
+end
+
+function DataStoreWriter:_computeValueDiff(original, incoming)
+	assert(original ~= DataStoreDeleteToken, "original cannot be DataStoreDeleteToken")
+	assert(incoming ~= DataStoreDeleteToken, "incoming cannot be DataStoreDeleteToken")
+
+	if original == incoming then
+		return nil
+	elseif original ~= nil and incoming == nil then
+		return DataStoreDeleteToken
+	elseif type(original) == "table" and type(incoming) == "table" then
+		return self:_computeTableDiff(original, incoming)
+	else
+		return incoming
+	end
+end
+
+function DataStoreWriter:_computeTableDiff(original, incoming)
+	assert(type(original) == "table", "Bad original")
+	assert(type(incoming) == "table", "Bad incoming")
+
+	local keys = Set.union(Set.fromKeys(original), Set.fromKeys(incoming))
+
+	local diffSnapshot = {}
+	for key, _ in pairs(keys) do
+		diffSnapshot[key] = self:_computeValueDiff(original[key], incoming[key])
+	end
+
+	if not DataStoreSnapshotUtils.isEmptySnapshot(diffSnapshot) then
+		return table.freeze(diffSnapshot)
+	else
+		if next(keys) then
+			return nil -- No delta
 		else
-			original[key] = result
+			return DataStoreDeleteToken
 		end
 	end
+end
+
+--[=[
+	Set of user ids to write with the data (only applies to top-level writer)
+
+	@param userIdList { number }
+]=]
+function DataStoreWriter:SetUserIdList(userIdList)
+	assert(type(userIdList) == "table" or userIdList == nil, "Bad userIdList")
+
+	self._userIdList = userIdList
+end
+
+--[=[
+	User ids to associate with data
+
+	@return userIdList { number }
+]=]
+function DataStoreWriter:GetUserIdList()
+	if self._userIdList == UNSET_TOKEN then
+		return nil
+	end
+
+	return self._userIdList
+end
+
+function DataStoreWriter:_writeMergeWriters(original)
+	local copy
+	if type(original) == "table" then
+		copy = table.clone(original)
+	else
+		copy = original
+	end
+
+	if next(self._writers) ~= nil then
+		-- Original was not a table. We need to swap to one.
+		if type(copy) ~= "table" then
+			copy = {}
+		end
+
+		-- Write our writers first...
+		for key, writer in pairs(self._writers) do
+			local result = writer:WriteMerge(copy[key])
+			if result == DataStoreDeleteToken then
+				copy[key] = nil
+			else
+				copy[key] = result
+			end
+		end
+	end
+
+	-- Write our save data next
+	if type(self._saveDataSnapshot) == "table" and next(self._saveDataSnapshot) ~= nil then
+		-- Original was not a table. We need to swap to one.
+		if type(copy) ~= "table" then
+			copy = {}
+		end
+
+		for key, value in pairs(self._saveDataSnapshot) do
+			if self._writers[key] then
+				warn(string.format("[DataStoreWriter._writeMergeWriters] - Overwriting key %q already saved as rawData with a writer with %q (was %q)", key, tostring(value), tostring(copy[key])))
+			end
+
+			if value == DataStoreDeleteToken then
+				copy[key] = nil
+			else
+				copy[key] = value
+			end
+		end
+	end
+
+	-- Handle empty table scenario..
+	-- This would also imply our original is nil somehow...
+	if type(copy) == "table" and next(copy) == nil then
+		if type(self._saveDataSnapshot) ~= "table" then
+			return nil
+		end
+	end
+
+	return copy
+end
+
+--[=[
+	Merges the new data into the original value
+
+	@param original any
+	@return any -- The original value
+]=]
+function DataStoreWriter:WriteMerge(original)
+	-- Prioritize save value first, followed by writers, followed by original value
+
+	if self._saveDataSnapshot == DataStoreDeleteToken then
+		return DataStoreDeleteToken
+	elseif self._saveDataSnapshot == UNSET_TOKEN or self._saveDataSnapshot == nil or type(self._saveDataSnapshot) == "table" then
+		return self:_writeMergeWriters(original)
+	else
+		-- Save data must be a boolean or something
+		return self._saveDataSnapshot
+	end
+end
+
+function DataStoreWriter:IsCompleteWipe()
+	if self._saveDataSnapshot == UNSET_TOKEN then
+		return false
+	end
+
+	if self._saveDataSnapshot == DataStoreDeleteToken then
+		return true
+	end
 
-	return original
+	return false
 end
 
 return DataStoreWriter

package/src/Server/PlayerDataStoreManager.lua

@@ -171,6 +171,7 @@ function PlayerDataStoreManager:_createDataStore(player)
 	assert(not self._datastores[player], "Bad player")
 
 	local datastore = DataStore.new(self._robloxDataStore, self:_getKey(player))
+	datastore:SetUserIdList({ player.UserId })
 
 	self._maid._savingConns[player] = datastore.Saving:Connect(function(promise)
 		self._pendingSaves:Add(promise)

package/src/Server/Utility/DataStorePromises.lua

@@ -69,13 +69,14 @@ function DataStorePromises.getAsync(robloxDataStore, key)
 
 	return Promise.spawn(function(resolve, reject)
 		local result = nil
+		local dataStoreKeyInfo = nil
 		local ok, err = pcall(function()
-			result = robloxDataStore:GetAsync(key)
+			result, dataStoreKeyInfo = robloxDataStore:GetAsync(key)
 		end)
 		if not ok then
 			return reject(err)
 		end
-		return resolve(result)
+		return resolve(result, dataStoreKeyInfo)
 	end)
 end
 

package/src/Shared/Utility/DataStoreStringUtils.lua

@@ -17,12 +17,16 @@ function DataStoreStringUtils.isValidUTF8(str)
 		return false, "Not a string"
 	end
 
-	-- https://gist.github.com/TheGreatSageEqualToHeaven/e0e1dc2698307c93f6013b9825705899?permalink_comment_id=4334757#gistcomment-4334757
-	if utf8.len(str) == nil then
+	if not utf8.len(str) then
+		return false, "Invalid string"
+	end
+
+	-- https://gist.github.com/TheGreatSageEqualToHeaven/e0e1dc2698307c93f6013b9825705899#patch-1
+	if string.match(str, "[^\0-\127]") then
 		return false, "Invalid string"
 	end
 
 	return true
 end
 
-return DataStoreStringUtils
+return DataStoreStringUtils

package/test/default.project.json

@@ -0,0 +1,21 @@
+{
+  "name": "DataStoreTest",
+  "tree": {
+    "$className": "DataModel",
+    "ServerScriptService": {
+      "datastore": {
+        "$path": ".."
+      },
+      "Script": {
+        "$path": "scripts/Server"
+      }
+    },
+    "StarterPlayer": {
+      "StarterPlayerScripts": {
+        "Main": {
+          "$path": "scripts/Client"
+        }
+      }
+    }
+  }
+}

package/test/scripts/Client/ClientMain.client.lua

@@ -0,0 +1,10 @@
+--[[
+	@class ClientMain
+]]
+
+local packages = game:GetService("ReplicatedStorage"):WaitForChild("Packages")
+
+local serviceBag = require(packages.ServiceBag).new()
+
+serviceBag:Init()
+serviceBag:Start()

package/test/scripts/Server/ServerMain.server.lua

@@ -0,0 +1,120 @@
+--[[
+	@class ServerMain
+]]
+
+local ServerScriptService = game:GetService("ServerScriptService")
+local HttpService = game:GetService("HttpService")
+
+local loader = ServerScriptService:FindFirstChild("LoaderUtils", true).Parent
+local packages = require(loader).bootstrapGame(ServerScriptService.datastore)
+
+local Maid = require(packages.Maid)
+local Promise = require(packages.Promise)
+
+local TURN_TIME = 8
+
+local function spinUpGameCopy(prefix)
+	assert(type(prefix) == "string", "Bad prefix")
+
+	local serviceBag = require(packages.ServiceBag).new()
+	serviceBag:GetService(require(packages.GameDataStoreService))
+	serviceBag:GetService(require(packages.PlayerDataStoreService))
+
+	serviceBag:Init()
+	serviceBag:Start()
+
+	local guid = prefix .. " " .. HttpService:GenerateGUID(false)
+	local maid = Maid.new()
+
+	local gameDataStore = serviceBag:GetService(require(packages.GameDataStoreService))
+	local bindToCloseService = serviceBag:GetService(require(packages.BindToCloseService))
+
+	-- This would be an aggressive usage of this area, it probably won't scale well enough.
+	-- But writing some shared code or something like API keys should scale fine.
+	maid:GivePromise(gameDataStore:PromiseDataStore()):Then(function(dataStore)
+		local substore = dataStore:GetSubStore("AliveServers")
+		substore:Store(guid, true)
+
+		-- maid:GiveTask(dataStore:Observe():Subscribe(function(viewSnapshot)
+		-- 	print(string.format("(%s) dataStore:Observe()", prefix), viewSnapshot)
+		-- end))
+
+		if prefix == "blue" then
+			dataStore:SetDoDebugWriting(true)
+			dataStore:SetSyncOnSave(true)
+			dataStore:SetAutoSaveTimeSeconds(4)
+
+			-- maid:GiveTask(dataStore:Observe():Subscribe(function(viewSnapshot)
+			-- 	print(string.format("(%s) dataStore:Observe()", prefix), viewSnapshot)
+			-- end))
+
+			task.delay(4*TURN_TIME, function()
+				warn("Blue server is restoring data")
+
+				substore:Store(guid, true)
+			end)
+		elseif prefix == "red" then
+			warn(string.format("%s server is storing data", prefix))
+
+			-- dataStore:SetDoDebugWriting(true)
+			dataStore:SetSyncOnSave(true)
+			dataStore:SetAutoSaveTimeSeconds(4)
+			-- dataStore:Save()
+
+			task.delay(TURN_TIME, function()
+				warn(string.format("%s server is wiping data", prefix))
+
+				substore:Wipe()
+				-- dataStore:Save()
+
+				task.delay(TURN_TIME, function()
+					warn(string.format("%s server is adding substore data", prefix))
+
+					substore:Store(guid, {
+						playerCount = 5;
+						startTime = DateTime.now().UnixTimestamp
+					})
+					-- dataStore:Save()
+
+					task.delay(TURN_TIME, function()
+						warn(string.format("%s server is changing player count", prefix))
+						local guidStore = substore:GetSubStore(guid)
+						guidStore:Store("playerCount", 25)
+						-- dataStore:Save()
+					end)
+				end)
+			end)
+		end
+
+		-- TODO: Update some random numbers every second for a while....
+
+		-- TODO: Force saving twice
+
+		maid:GiveTask(dataStore:Observe():Subscribe(function(viewSnapshot)
+			print(string.format("(%s) dataStore:Observe()", prefix), viewSnapshot)
+		end))
+
+		-- dataStore:LoadAll():Then(function(data)
+		-- 	-- print(string.format("[%s][LoadAll] - Load all", prefix), data)
+		-- end)
+
+		-- local entrySubstore = substore:GetSubStore(guid)
+		-- entrySubstore:LoadAll():Then(function(data)
+		-- 	-- print(string.format("[%s][SUBSTORE][LoadAll] Loaded substore", prefix), data)
+		-- end)
+
+		-- entrySubstore:Overwrite(os.clock())
+
+		maid:GiveTask(bindToCloseService:RegisterPromiseOnCloseCallback(function()
+			substore:Delete(guid)
+			return Promise.resolved()
+		end))
+
+	end)
+
+	return maid
+end
+
+spinUpGameCopy("red")
+spinUpGameCopy("blue")
+spinUpGameCopy("green")